drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c

   1 /*
   2  * Copyright 2023 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <linux/delay.h>
  24 #include <linux/kernel.h>
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_gfx.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_smu.h"
  32 #include "amdgpu_atomfirmware.h"
  33 #include "imu_v12_0.h"
  34 #include "soc24.h"
  35 #include "nvd.h"
  36
  37 #include "gc/gc_12_0_0_offset.h"
  38 #include "gc/gc_12_0_0_sh_mask.h"
  39 #include "soc24_enum.h"
  40 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
  41
  42 #include "soc15.h"
  43 #include "soc15d.h"
  44 #include "clearstate_gfx12.h"
  45 #include "v12_structs.h"
  46 #include "gfx_v12_0.h"
  47 #include "nbif_v6_3_1.h"
  48 #include "mes_v12_0.h"
  49
  50 #define GFX12_NUM_GFX_RINGS     1
  51 #define GFX12_MEC_HPD_SIZE      2048
  52
  53 #define RLCG_UCODE_LOADING_START_ADDRESS        0x00002000L
  54
  55 MODULE_FIRMWARE("amdgpu/gc_12_0_0_pfp.bin");
  56 MODULE_FIRMWARE("amdgpu/gc_12_0_0_me.bin");
  57 MODULE_FIRMWARE("amdgpu/gc_12_0_0_mec.bin");
  58 MODULE_FIRMWARE("amdgpu/gc_12_0_0_rlc.bin");
  59 MODULE_FIRMWARE("amdgpu/gc_12_0_0_toc.bin");
  60 MODULE_FIRMWARE("amdgpu/gc_12_0_1_pfp.bin");
  61 MODULE_FIRMWARE("amdgpu/gc_12_0_1_me.bin");
  62 MODULE_FIRMWARE("amdgpu/gc_12_0_1_mec.bin");
  63 MODULE_FIRMWARE("amdgpu/gc_12_0_1_rlc.bin");
  64 MODULE_FIRMWARE("amdgpu/gc_12_0_1_toc.bin");
  65
  66 #define DEFAULT_SH_MEM_CONFIG \
  67         ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
  68          (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
  69          (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
  70
  71 static void gfx_v12_0_disable_gpa_mode(struct amdgpu_device *adev);
  72 static void gfx_v12_0_set_ring_funcs(struct amdgpu_device *adev);
  73 static void gfx_v12_0_set_irq_funcs(struct amdgpu_device *adev);
  74 static void gfx_v12_0_set_rlc_funcs(struct amdgpu_device *adev);
  75 static void gfx_v12_0_set_mqd_funcs(struct amdgpu_device *adev);
  76 static void gfx_v12_0_set_imu_funcs(struct amdgpu_device *adev);
  77 static int gfx_v12_0_get_cu_info(struct amdgpu_device *adev,
  78                                  struct amdgpu_cu_info *cu_info);
  79 static uint64_t gfx_v12_0_get_gpu_clock_counter(struct amdgpu_device *adev);
  80 static void gfx_v12_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
  81                                    u32 sh_num, u32 instance, int xcc_id);
  82 static u32 gfx_v12_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
  83
  84 static void gfx_v12_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
  85 static void gfx_v12_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
  86                                      uint32_t val);
  87 static int gfx_v12_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
  88 static void gfx_v12_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
  89                                            uint16_t pasid, uint32_t flush_type,
  90                                            bool all_hub, uint8_t dst_sel);
  91 static void gfx_v12_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
  92 static void gfx_v12_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
  93 static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev,
  94                                       bool enable);
  95
  96 static void gfx_v12_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
  97                                         uint64_t queue_mask)
  98 {
  99         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 100         amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
 101                           PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
 102         amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
 103         amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
 104         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 105         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 106         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 107         amdgpu_ring_write(kiq_ring, 0);
 108 }
 109
 110 static void gfx_v12_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 111                                      struct amdgpu_ring *ring)
 112 {
 113         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 114         uint64_t wptr_addr = ring->wptr_gpu_addr;
 115         uint32_t me = 0, eng_sel = 0;
 116
 117         switch (ring->funcs->type) {
 118         case AMDGPU_RING_TYPE_COMPUTE:
 119                 me = 1;
 120                 eng_sel = 0;
 121                 break;
 122         case AMDGPU_RING_TYPE_GFX:
 123                 me = 0;
 124                 eng_sel = 4;
 125                 break;
 126         case AMDGPU_RING_TYPE_MES:
 127                 me = 2;
 128                 eng_sel = 5;
 129                 break;
 130         default:
 131                 WARN_ON(1);
 132         }
 133
 134         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 135         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 136         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 137                           PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 138                           PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 139                           PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 140                           PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 141                           PACKET3_MAP_QUEUES_ME((me)) |
 142                           PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
 143                           PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
 144                           PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 145                           PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
 146         amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 147         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 148         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 149         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 150         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 151 }
 152
 153 static void gfx_v12_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 154                                        struct amdgpu_ring *ring,
 155                                        enum amdgpu_unmap_queues_action action,
 156                                        u64 gpu_addr, u64 seq)
 157 {
 158         struct amdgpu_device *adev = kiq_ring->adev;
 159         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 160
 161         if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
 162                 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
 163                 return;
 164         }
 165
 166         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 167         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 168                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 169                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 170                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 171                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 172         amdgpu_ring_write(kiq_ring,
 173                   PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 174
 175         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 176                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 177                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 178                 amdgpu_ring_write(kiq_ring, seq);
 179         } else {
 180                 amdgpu_ring_write(kiq_ring, 0);
 181                 amdgpu_ring_write(kiq_ring, 0);
 182                 amdgpu_ring_write(kiq_ring, 0);
 183         }
 184 }
 185
 186 static void gfx_v12_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 187                                        struct amdgpu_ring *ring,
 188                                        u64 addr, u64 seq)
 189 {
 190         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 191
 192         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 193         amdgpu_ring_write(kiq_ring,
 194                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 195                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 196                           PACKET3_QUERY_STATUS_COMMAND(2));
 197         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 198                           PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 199                           PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 200         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 201         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 202         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 203         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 204 }
 205
 206 static void gfx_v12_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 207                                           uint16_t pasid,
 208                                           uint32_t flush_type,
 209                                           bool all_hub)
 210 {
 211         gfx_v12_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
 212 }
 213
 214 static const struct kiq_pm4_funcs gfx_v12_0_kiq_pm4_funcs = {
 215         .kiq_set_resources = gfx_v12_0_kiq_set_resources,
 216         .kiq_map_queues = gfx_v12_0_kiq_map_queues,
 217         .kiq_unmap_queues = gfx_v12_0_kiq_unmap_queues,
 218         .kiq_query_status = gfx_v12_0_kiq_query_status,
 219         .kiq_invalidate_tlbs = gfx_v12_0_kiq_invalidate_tlbs,
 220         .set_resources_size = 8,
 221         .map_queues_size = 7,
 222         .unmap_queues_size = 6,
 223         .query_status_size = 7,
 224         .invalidate_tlbs_size = 2,
 225 };
 226
 227 static void gfx_v12_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 228 {
 229         adev->gfx.kiq[0].pmf = &gfx_v12_0_kiq_pm4_funcs;
 230 }
 231
 232 static void gfx_v12_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 233                                    int mem_space, int opt, uint32_t addr0,
 234                                    uint32_t addr1, uint32_t ref,
 235                                    uint32_t mask, uint32_t inv)
 236 {
 237         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 238         amdgpu_ring_write(ring,
 239                           /* memory (1) or register (0) */
 240                           (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 241                            WAIT_REG_MEM_OPERATION(opt) | /* wait */
 242                            WAIT_REG_MEM_FUNCTION(3) |  /* equal */
 243                            WAIT_REG_MEM_ENGINE(eng_sel)));
 244
 245         if (mem_space)
 246                 BUG_ON(addr0 & 0x3); /* Dword align */
 247         amdgpu_ring_write(ring, addr0);
 248         amdgpu_ring_write(ring, addr1);
 249         amdgpu_ring_write(ring, ref);
 250         amdgpu_ring_write(ring, mask);
 251         amdgpu_ring_write(ring, inv); /* poll interval */
 252 }
 253
 254 static int gfx_v12_0_ring_test_ring(struct amdgpu_ring *ring)
 255 {
 256         struct amdgpu_device *adev = ring->adev;
 257         uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 258         uint32_t tmp = 0;
 259         unsigned i;
 260         int r;
 261
 262         WREG32(scratch, 0xCAFEDEAD);
 263         r = amdgpu_ring_alloc(ring, 5);
 264         if (r) {
 265                 dev_err(adev->dev,
 266                         "amdgpu: cp failed to lock ring %d (%d).\n",
 267                         ring->idx, r);
 268                 return r;
 269         }
 270
 271         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
 272                 gfx_v12_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
 273         } else {
 274                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 275                 amdgpu_ring_write(ring, scratch -
 276                                   PACKET3_SET_UCONFIG_REG_START);
 277                 amdgpu_ring_write(ring, 0xDEADBEEF);
 278         }
 279         amdgpu_ring_commit(ring);
 280
 281         for (i = 0; i < adev->usec_timeout; i++) {
 282                 tmp = RREG32(scratch);
 283                 if (tmp == 0xDEADBEEF)
 284                         break;
 285                 if (amdgpu_emu_mode == 1)
 286                         msleep(1);
 287                 else
 288                         udelay(1);
 289         }
 290
 291         if (i >= adev->usec_timeout)
 292                 r = -ETIMEDOUT;
 293         return r;
 294 }
 295
 296 static int gfx_v12_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 297 {
 298         struct amdgpu_device *adev = ring->adev;
 299         struct amdgpu_ib ib;
 300         struct dma_fence *f = NULL;
 301         unsigned index;
 302         uint64_t gpu_addr;
 303         volatile uint32_t *cpu_ptr;
 304         long r;
 305
 306         /* MES KIQ fw hasn't indirect buffer support for now */
 307         if (adev->enable_mes_kiq &&
 308             ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 309                 return 0;
 310
 311         memset(&ib, 0, sizeof(ib));
 312
 313         if (ring->is_mes_queue) {
 314                 uint32_t padding, offset;
 315
 316                 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
 317                 padding = amdgpu_mes_ctx_get_offs(ring,
 318                                                   AMDGPU_MES_CTX_PADDING_OFFS);
 319
 320                 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 321                 ib.ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 322
 323                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, padding);
 324                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, padding);
 325                 *cpu_ptr = cpu_to_le32(0xCAFEDEAD);
 326         } else {
 327                 r = amdgpu_device_wb_get(adev, &index);
 328                 if (r)
 329                         return r;
 330
 331                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 332                 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 333                 cpu_ptr = &adev->wb.wb[index];
 334
 335                 r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
 336                 if (r) {
 337                         dev_err(adev->dev, "amdgpu: failed to get ib (%ld).\n", r);
 338                         goto err1;
 339                 }
 340         }
 341
 342         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
 343         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
 344         ib.ptr[2] = lower_32_bits(gpu_addr);
 345         ib.ptr[3] = upper_32_bits(gpu_addr);
 346         ib.ptr[4] = 0xDEADBEEF;
 347         ib.length_dw = 5;
 348
 349         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 350         if (r)
 351                 goto err2;
 352
 353         r = dma_fence_wait_timeout(f, false, timeout);
 354         if (r == 0) {
 355                 r = -ETIMEDOUT;
 356                 goto err2;
 357         } else if (r < 0) {
 358                 goto err2;
 359         }
 360
 361         if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
 362                 r = 0;
 363         else
 364                 r = -EINVAL;
 365 err2:
 366         if (!ring->is_mes_queue)
 367                 amdgpu_ib_free(adev, &ib, NULL);
 368         dma_fence_put(f);
 369 err1:
 370         if (!ring->is_mes_queue)
 371                 amdgpu_device_wb_free(adev, index);
 372         return r;
 373 }
 374
 375 static void gfx_v12_0_free_microcode(struct amdgpu_device *adev)
 376 {
 377         amdgpu_ucode_release(&adev->gfx.pfp_fw);
 378         amdgpu_ucode_release(&adev->gfx.me_fw);
 379         amdgpu_ucode_release(&adev->gfx.rlc_fw);
 380         amdgpu_ucode_release(&adev->gfx.mec_fw);
 381
 382         kfree(adev->gfx.rlc.register_list_format);
 383 }
 384
 385 static int gfx_v12_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
 386 {
 387         const struct psp_firmware_header_v1_0 *toc_hdr;
 388         int err = 0;
 389         char fw_name[40];
 390
 391         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_toc.bin", ucode_prefix);
 392         err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, fw_name);
 393         if (err)
 394                 goto out;
 395
 396         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
 397         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
 398         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
 399         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
 400         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
 401                         le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
 402         return 0;
 403 out:
 404         amdgpu_ucode_release(&adev->psp.toc_fw);
 405         return err;
 406 }
 407
 408 static int gfx_v12_0_init_microcode(struct amdgpu_device *adev)
 409 {
 410         char fw_name[40];
 411         char ucode_prefix[15];
 412         int err;
 413         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 414         uint16_t version_major;
 415         uint16_t version_minor;
 416
 417         DRM_DEBUG("\n");
 418
 419         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
 420
 421         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", ucode_prefix);
 422         err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, fw_name);
 423         if (err)
 424                 goto out;
 425         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
 426         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
 427
 428         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", ucode_prefix);
 429         err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, fw_name);
 430         if (err)
 431                 goto out;
 432         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
 433         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
 434
 435         if (!amdgpu_sriov_vf(adev)) {
 436                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", ucode_prefix);
 437                 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, fw_name);
 438                 if (err)
 439                         goto out;
 440                 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 441                 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 442                 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
 443                 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
 444                 if (err)
 445                         goto out;
 446         }
 447
 448         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", ucode_prefix);
 449         err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name);
 450         if (err)
 451                 goto out;
 452         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
 453         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
 454         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
 455
 456         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
 457                 err = gfx_v12_0_init_toc_microcode(adev, ucode_prefix);
 458
 459         /* only one MEC for gfx 12 */
 460         adev->gfx.mec2_fw = NULL;
 461
 462         if (adev->gfx.imu.funcs) {
 463                 if (adev->gfx.imu.funcs->init_microcode) {
 464                         err = adev->gfx.imu.funcs->init_microcode(adev);
 465                         if (err)
 466                                 dev_err(adev->dev, "Failed to load imu firmware!\n");
 467                 }
 468         }
 469
 470 out:
 471         if (err) {
 472                 amdgpu_ucode_release(&adev->gfx.pfp_fw);
 473                 amdgpu_ucode_release(&adev->gfx.me_fw);
 474                 amdgpu_ucode_release(&adev->gfx.rlc_fw);
 475                 amdgpu_ucode_release(&adev->gfx.mec_fw);
 476         }
 477
 478         return err;
 479 }
 480
 481 static u32 gfx_v12_0_get_csb_size(struct amdgpu_device *adev)
 482 {
 483         u32 count = 0;
 484         const struct cs_section_def *sect = NULL;
 485         const struct cs_extent_def *ext = NULL;
 486
 487         count += 1;
 488
 489         for (sect = gfx12_cs_data; sect->section != NULL; ++sect) {
 490                 if (sect->id == SECT_CONTEXT) {
 491                         for (ext = sect->section; ext->extent != NULL; ++ext)
 492                                 count += 2 + ext->reg_count;
 493                 } else
 494                         return 0;
 495         }
 496
 497         return count;
 498 }
 499
 500 static void gfx_v12_0_get_csb_buffer(struct amdgpu_device *adev,
 501                                      volatile u32 *buffer)
 502 {
 503         u32 count = 0, clustercount = 0, i;
 504         const struct cs_section_def *sect = NULL;
 505         const struct cs_extent_def *ext = NULL;
 506
 507         if (adev->gfx.rlc.cs_data == NULL)
 508                 return;
 509         if (buffer == NULL)
 510                 return;
 511
 512         count += 1;
 513
 514         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
 515                 if (sect->id == SECT_CONTEXT) {
 516                         for (ext = sect->section; ext->extent != NULL; ++ext) {
 517                                 clustercount++;
 518                                 buffer[count++] = ext->reg_count;
 519                                 buffer[count++] = ext->reg_index;
 520
 521                                 for (i = 0; i < ext->reg_count; i++)
 522                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
 523                         }
 524                 } else
 525                         return;
 526         }
 527
 528         buffer[0] = clustercount;
 529 }
 530
 531 static void gfx_v12_0_rlc_fini(struct amdgpu_device *adev)
 532 {
 533         /* clear state block */
 534         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
 535                         &adev->gfx.rlc.clear_state_gpu_addr,
 536                         (void **)&adev->gfx.rlc.cs_ptr);
 537
 538         /* jump table block */
 539         amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
 540                         &adev->gfx.rlc.cp_table_gpu_addr,
 541                         (void **)&adev->gfx.rlc.cp_table_ptr);
 542 }
 543
 544 static void gfx_v12_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
 545 {
 546         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
 547
 548         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
 549         reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 550         reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
 551         reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
 552         reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
 553         reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
 554         reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
 555         reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
 556         adev->gfx.rlc.rlcg_reg_access_supported = true;
 557 }
 558
 559 static int gfx_v12_0_rlc_init(struct amdgpu_device *adev)
 560 {
 561         const struct cs_section_def *cs_data;
 562         int r;
 563
 564         adev->gfx.rlc.cs_data = gfx12_cs_data;
 565
 566         cs_data = adev->gfx.rlc.cs_data;
 567
 568         if (cs_data) {
 569                 /* init clear state block */
 570                 r = amdgpu_gfx_rlc_init_csb(adev);
 571                 if (r)
 572                         return r;
 573         }
 574
 575         /* init spm vmid with 0xf */
 576         if (adev->gfx.rlc.funcs->update_spm_vmid)
 577                 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
 578
 579         return 0;
 580 }
 581
 582 static void gfx_v12_0_mec_fini(struct amdgpu_device *adev)
 583 {
 584         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
 585         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
 586         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
 587 }
 588
 589 static void gfx_v12_0_me_init(struct amdgpu_device *adev)
 590 {
 591         bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 592
 593         amdgpu_gfx_graphics_queue_acquire(adev);
 594 }
 595
 596 static int gfx_v12_0_mec_init(struct amdgpu_device *adev)
 597 {
 598         int r;
 599         u32 *hpd;
 600         size_t mec_hpd_size;
 601
 602         bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 603
 604         /* take ownership of the relevant compute queues */
 605         amdgpu_gfx_compute_queue_acquire(adev);
 606         mec_hpd_size = adev->gfx.num_compute_rings * GFX12_MEC_HPD_SIZE;
 607
 608         if (mec_hpd_size) {
 609                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
 610                                               AMDGPU_GEM_DOMAIN_GTT,
 611                                               &adev->gfx.mec.hpd_eop_obj,
 612                                               &adev->gfx.mec.hpd_eop_gpu_addr,
 613                                               (void **)&hpd);
 614                 if (r) {
 615                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
 616                         gfx_v12_0_mec_fini(adev);
 617                         return r;
 618                 }
 619
 620                 memset(hpd, 0, mec_hpd_size);
 621
 622                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
 623                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
 624         }
 625
 626         return 0;
 627 }
 628
 629 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
 630 {
 631         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 632                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 633                 (address << SQ_IND_INDEX__INDEX__SHIFT));
 634         return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 635 }
 636
 637 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
 638                            uint32_t thread, uint32_t regno,
 639                            uint32_t num, uint32_t *out)
 640 {
 641         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 642                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 643                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
 644                 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
 645                 (SQ_IND_INDEX__AUTO_INCR_MASK));
 646         while (num--)
 647                 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 648 }
 649
 650 static void gfx_v12_0_read_wave_data(struct amdgpu_device *adev,
 651                                      uint32_t xcc_id,
 652                                      uint32_t simd, uint32_t wave,
 653                                      uint32_t *dst, int *no_fields)
 654 {
 655         /* in gfx12 the SIMD_ID is specified as part of the INSTANCE
 656          * field when performing a select_se_sh so it should be
 657          * zero here */
 658         WARN_ON(simd != 0);
 659
 660         /* type 4 wave data */
 661         dst[(*no_fields)++] = 4;
 662         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
 663         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
 664         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
 665         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
 666         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
 667         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
 668         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
 669         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
 670         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
 671         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
 672         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
 673         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
 674         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
 675         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
 676         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATE_PRIV);
 677         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXCP_FLAG_PRIV);
 678         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXCP_FLAG_USER);
 679         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAP_CTRL);
 680         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_ACTIVE);
 681         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_VALID_AND_IDLE);
 682         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_DVGPR_ALLOC_LO);
 683         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_DVGPR_ALLOC_HI);
 684         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_SCHED_MODE);
 685 }
 686
 687 static void gfx_v12_0_read_wave_sgprs(struct amdgpu_device *adev,
 688                                       uint32_t xcc_id, uint32_t simd,
 689                                       uint32_t wave, uint32_t start,
 690                                       uint32_t size, uint32_t *dst)
 691 {
 692         WARN_ON(simd != 0);
 693
 694         wave_read_regs(
 695                 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
 696                 dst);
 697 }
 698
 699 static void gfx_v12_0_read_wave_vgprs(struct amdgpu_device *adev,
 700                                       uint32_t xcc_id, uint32_t simd,
 701                                       uint32_t wave, uint32_t thread,
 702                                       uint32_t start, uint32_t size,
 703                                       uint32_t *dst)
 704 {
 705         wave_read_regs(
 706                 adev, wave, thread,
 707                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
 708 }
 709
 710 static void gfx_v12_0_select_me_pipe_q(struct amdgpu_device *adev,
 711                                        u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
 712 {
 713         soc24_grbm_select(adev, me, pipe, q, vm);
 714 }
 715
 716 static const struct amdgpu_gfx_funcs gfx_v12_0_gfx_funcs = {
 717         .get_gpu_clock_counter = &gfx_v12_0_get_gpu_clock_counter,
 718         .select_se_sh = &gfx_v12_0_select_se_sh,
 719         .read_wave_data = &gfx_v12_0_read_wave_data,
 720         .read_wave_sgprs = &gfx_v12_0_read_wave_sgprs,
 721         .read_wave_vgprs = &gfx_v12_0_read_wave_vgprs,
 722         .select_me_pipe_q = &gfx_v12_0_select_me_pipe_q,
 723         .update_perfmon_mgcg = &gfx_v12_0_update_perf_clk,
 724 };
 725
 726 static int gfx_v12_0_gpu_early_init(struct amdgpu_device *adev)
 727 {
 728
 729         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 730         case IP_VERSION(12, 0, 0):
 731         case IP_VERSION(12, 0, 1):
 732                 adev->gfx.config.max_hw_contexts = 8;
 733                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
 734                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
 735                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
 736                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
 737                 break;
 738         default:
 739                 BUG();
 740                 break;
 741         }
 742
 743         return 0;
 744 }
 745
 746 static int gfx_v12_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
 747                                    int me, int pipe, int queue)
 748 {
 749         int r;
 750         struct amdgpu_ring *ring;
 751         unsigned int irq_type;
 752
 753         ring = &adev->gfx.gfx_ring[ring_id];
 754
 755         ring->me = me;
 756         ring->pipe = pipe;
 757         ring->queue = queue;
 758
 759         ring->ring_obj = NULL;
 760         ring->use_doorbell = true;
 761
 762         if (!ring_id)
 763                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
 764         else
 765                 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
 766         ring->vm_hub = AMDGPU_GFXHUB(0);
 767         sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 768
 769         irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
 770         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
 771                              AMDGPU_RING_PRIO_DEFAULT, NULL);
 772         if (r)
 773                 return r;
 774         return 0;
 775 }
 776
 777 static int gfx_v12_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
 778                                        int mec, int pipe, int queue)
 779 {
 780         int r;
 781         unsigned irq_type;
 782         struct amdgpu_ring *ring;
 783         unsigned int hw_prio;
 784
 785         ring = &adev->gfx.compute_ring[ring_id];
 786
 787         /* mec0 is me1 */
 788         ring->me = mec + 1;
 789         ring->pipe = pipe;
 790         ring->queue = queue;
 791
 792         ring->ring_obj = NULL;
 793         ring->use_doorbell = true;
 794         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
 795         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
 796                                 + (ring_id * GFX12_MEC_HPD_SIZE);
 797         ring->vm_hub = AMDGPU_GFXHUB(0);
 798         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 799
 800         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
 801                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
 802                 + ring->pipe;
 803         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
 804                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
 805         /* type-2 packets are deprecated on MEC, use type-3 instead */
 806         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
 807                              hw_prio, NULL);
 808         if (r)
 809                 return r;
 810
 811         return 0;
 812 }
 813
 814 static struct {
 815         SOC24_FIRMWARE_ID       id;
 816         unsigned int            offset;
 817         unsigned int            size;
 818         unsigned int            size_x16;
 819 } rlc_autoload_info[SOC24_FIRMWARE_ID_MAX];
 820
 821 #define RLC_TOC_OFFSET_DWUNIT   8
 822 #define RLC_SIZE_MULTIPLE       1024
 823 #define RLC_TOC_UMF_SIZE_inM    23ULL
 824 #define RLC_TOC_FORMAT_API      165ULL
 825
 826 static void gfx_v12_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
 827 {
 828         RLC_TABLE_OF_CONTENT_V2 *ucode = rlc_toc;
 829
 830         while (ucode && (ucode->id > SOC24_FIRMWARE_ID_INVALID)) {
 831                 rlc_autoload_info[ucode->id].id = ucode->id;
 832                 rlc_autoload_info[ucode->id].offset =
 833                         ucode->offset * RLC_TOC_OFFSET_DWUNIT * 4;
 834                 rlc_autoload_info[ucode->id].size =
 835                         ucode->size_x16 ? ucode->size * RLC_SIZE_MULTIPLE * 4 :
 836                                           ucode->size * 4;
 837                 ucode++;
 838         }
 839 }
 840
 841 static uint32_t gfx_v12_0_calc_toc_total_size(struct amdgpu_device *adev)
 842 {
 843         uint32_t total_size = 0;
 844         SOC24_FIRMWARE_ID id;
 845
 846         gfx_v12_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
 847
 848         for (id = SOC24_FIRMWARE_ID_RLC_G_UCODE; id < SOC24_FIRMWARE_ID_MAX; id++)
 849                 total_size += rlc_autoload_info[id].size;
 850
 851         /* In case the offset in rlc toc ucode is aligned */
 852         if (total_size < rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset)
 853                 total_size = rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset +
 854                         rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].size;
 855         if (total_size < (RLC_TOC_UMF_SIZE_inM << 20))
 856                 total_size = RLC_TOC_UMF_SIZE_inM << 20;
 857
 858         return total_size;
 859 }
 860
 861 static int gfx_v12_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
 862 {
 863         int r;
 864         uint32_t total_size;
 865
 866         total_size = gfx_v12_0_calc_toc_total_size(adev);
 867
 868         r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
 869                                       AMDGPU_GEM_DOMAIN_VRAM,
 870                                       &adev->gfx.rlc.rlc_autoload_bo,
 871                                       &adev->gfx.rlc.rlc_autoload_gpu_addr,
 872                                       (void **)&adev->gfx.rlc.rlc_autoload_ptr);
 873
 874         if (r) {
 875                 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
 876                 return r;
 877         }
 878
 879         return 0;
 880 }
 881
 882 static void gfx_v12_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
 883                                                        SOC24_FIRMWARE_ID id,
 884                                                        const void *fw_data,
 885                                                        uint32_t fw_size)
 886 {
 887         uint32_t toc_offset;
 888         uint32_t toc_fw_size;
 889         char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
 890
 891         if (id <= SOC24_FIRMWARE_ID_INVALID || id >= SOC24_FIRMWARE_ID_MAX)
 892                 return;
 893
 894         toc_offset = rlc_autoload_info[id].offset;
 895         toc_fw_size = rlc_autoload_info[id].size;
 896
 897         if (fw_size == 0)
 898                 fw_size = toc_fw_size;
 899
 900         if (fw_size > toc_fw_size)
 901                 fw_size = toc_fw_size;
 902
 903         memcpy(ptr + toc_offset, fw_data, fw_size);
 904
 905         if (fw_size < toc_fw_size)
 906                 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
 907 }
 908
 909 static void
 910 gfx_v12_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev)
 911 {
 912         void *data;
 913         uint32_t size;
 914         uint32_t *toc_ptr;
 915
 916         data = adev->psp.toc.start_addr;
 917         size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_TOC].size;
 918
 919         toc_ptr = (uint32_t *)data + size / 4 - 2;
 920         *toc_ptr = (RLC_TOC_FORMAT_API << 24) | 0x1;
 921
 922         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_TOC,
 923                                                    data, size);
 924 }
 925
 926 static void
 927 gfx_v12_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev)
 928 {
 929         const __le32 *fw_data;
 930         uint32_t fw_size;
 931         const struct gfx_firmware_header_v2_0 *cpv2_hdr;
 932         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 933         const struct rlc_firmware_header_v2_1 *rlcv21_hdr;
 934         const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
 935         uint16_t version_major, version_minor;
 936
 937         /* pfp ucode */
 938         cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
 939                 adev->gfx.pfp_fw->data;
 940         /* instruction */
 941         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
 942                 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
 943         fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
 944         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_PFP,
 945                                                    fw_data, fw_size);
 946         /* data */
 947         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
 948                 le32_to_cpu(cpv2_hdr->data_offset_bytes));
 949         fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
 950         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_PFP_P0_STACK,
 951                                                    fw_data, fw_size);
 952         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_PFP_P1_STACK,
 953                                                    fw_data, fw_size);
 954         /* me ucode */
 955         cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
 956                 adev->gfx.me_fw->data;
 957         /* instruction */
 958         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
 959                 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
 960         fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
 961         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_ME,
 962                                                    fw_data, fw_size);
 963         /* data */
 964         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
 965                 le32_to_cpu(cpv2_hdr->data_offset_bytes));
 966         fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
 967         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_ME_P0_STACK,
 968                                                    fw_data, fw_size);
 969         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_ME_P1_STACK,
 970                                                    fw_data, fw_size);
 971         /* mec ucode */
 972         cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
 973                 adev->gfx.mec_fw->data;
 974         /* instruction */
 975         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
 976                 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
 977         fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
 978         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC,
 979                                                    fw_data, fw_size);
 980         /* data */
 981         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
 982                 le32_to_cpu(cpv2_hdr->data_offset_bytes));
 983         fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
 984         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P0_STACK,
 985                                                    fw_data, fw_size);
 986         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P1_STACK,
 987                                                    fw_data, fw_size);
 988         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P2_STACK,
 989                                                    fw_data, fw_size);
 990         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P3_STACK,
 991                                                    fw_data, fw_size);
 992
 993         /* rlc ucode */
 994         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
 995                 adev->gfx.rlc_fw->data;
 996         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
 997                         le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
 998         fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
 999         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_G_UCODE,
1000                                                    fw_data, fw_size);
1001
1002         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1003         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1004         if (version_major == 2) {
1005                 if (version_minor >= 1) {
1006                         rlcv21_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1007
1008                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1009                                         le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_offset_bytes));
1010                         fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_size_bytes);
1011                         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLCG_SCRATCH,
1012                                                    fw_data, fw_size);
1013
1014                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1015                                         le32_to_cpu(rlcv21_hdr->save_restore_list_srm_offset_bytes));
1016                         fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_srm_size_bytes);
1017                         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_SRM_ARAM,
1018                                                    fw_data, fw_size);
1019                 }
1020                 if (version_minor >= 2) {
1021                         rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1022
1023                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1024                                         le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1025                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1026                         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_UCODE,
1027                                                    fw_data, fw_size);
1028
1029                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1030                                         le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1031                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1032                         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_DRAM_BOOT,
1033                                                    fw_data, fw_size);
1034                 }
1035         }
1036 }
1037
1038 static void
1039 gfx_v12_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev)
1040 {
1041         const __le32 *fw_data;
1042         uint32_t fw_size;
1043         const struct sdma_firmware_header_v3_0 *sdma_hdr;
1044
1045         sdma_hdr = (const struct sdma_firmware_header_v3_0 *)
1046                 adev->sdma.instance[0].fw->data;
1047         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1048                         le32_to_cpu(sdma_hdr->ucode_offset_bytes));
1049         fw_size = le32_to_cpu(sdma_hdr->ucode_size_bytes);
1050
1051         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_SDMA_UCODE_TH0,
1052                                                    fw_data, fw_size);
1053 }
1054
1055 static void
1056 gfx_v12_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev)
1057 {
1058         const __le32 *fw_data;
1059         unsigned fw_size;
1060         const struct mes_firmware_header_v1_0 *mes_hdr;
1061         int pipe, ucode_id, data_id;
1062
1063         for (pipe = 0; pipe < 2; pipe++) {
1064                 if (pipe == 0) {
1065                         ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P0;
1066                         data_id  = SOC24_FIRMWARE_ID_RS64_MES_P0_STACK;
1067                 } else {
1068                         ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P1;
1069                         data_id  = SOC24_FIRMWARE_ID_RS64_MES_P1_STACK;
1070                 }
1071
1072                 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1073                         adev->mes.fw[pipe]->data;
1074
1075                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1076                                 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1077                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1078
1079                 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, ucode_id, fw_data, fw_size);
1080
1081                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1082                                 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1083                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1084
1085                 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, data_id, fw_data, fw_size);
1086         }
1087 }
1088
1089 static int gfx_v12_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1090 {
1091         uint32_t rlc_g_offset, rlc_g_size;
1092         uint64_t gpu_addr;
1093         uint32_t data;
1094
1095         /* RLC autoload sequence 2: copy ucode */
1096         gfx_v12_0_rlc_backdoor_autoload_copy_sdma_ucode(adev);
1097         gfx_v12_0_rlc_backdoor_autoload_copy_gfx_ucode(adev);
1098         gfx_v12_0_rlc_backdoor_autoload_copy_mes_ucode(adev);
1099         gfx_v12_0_rlc_backdoor_autoload_copy_toc_ucode(adev);
1100
1101         rlc_g_offset = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].offset;
1102         rlc_g_size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].size;
1103         gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset - adev->gmc.vram_start;
1104
1105         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1106         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1107
1108         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1109
1110         if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
1111                 /* RLC autoload sequence 3: load IMU fw */
1112                 if (adev->gfx.imu.funcs->load_microcode)
1113                         adev->gfx.imu.funcs->load_microcode(adev);
1114                 /* RLC autoload sequence 4 init IMU fw */
1115                 if (adev->gfx.imu.funcs->setup_imu)
1116                         adev->gfx.imu.funcs->setup_imu(adev);
1117                 if (adev->gfx.imu.funcs->start_imu)
1118                         adev->gfx.imu.funcs->start_imu(adev);
1119
1120                 /* RLC autoload sequence 5 disable gpa mode */
1121                 gfx_v12_0_disable_gpa_mode(adev);
1122         } else {
1123                 /* unhalt rlc to start autoload without imu */
1124                 data = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
1125                 data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD0_ENABLE, 1);
1126                 data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
1127                 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, data);
1128                 WREG32_SOC15(GC, 0, regRLC_CNTL, RLC_CNTL__RLC_ENABLE_F32_MASK);
1129         }
1130
1131         return 0;
1132 }
1133
1134 static int gfx_v12_0_sw_init(void *handle)
1135 {
1136         int i, j, k, r, ring_id = 0;
1137         unsigned num_compute_rings;
1138         int xcc_id = 0;
1139         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1140
1141         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1142         case IP_VERSION(12, 0, 0):
1143         case IP_VERSION(12, 0, 1):
1144                 adev->gfx.me.num_me = 1;
1145                 adev->gfx.me.num_pipe_per_me = 1;
1146                 adev->gfx.me.num_queue_per_pipe = 1;
1147                 adev->gfx.mec.num_mec = 2;
1148                 adev->gfx.mec.num_pipe_per_mec = 2;
1149                 adev->gfx.mec.num_queue_per_pipe = 4;
1150                 break;
1151         default:
1152                 adev->gfx.me.num_me = 1;
1153                 adev->gfx.me.num_pipe_per_me = 1;
1154                 adev->gfx.me.num_queue_per_pipe = 1;
1155                 adev->gfx.mec.num_mec = 1;
1156                 adev->gfx.mec.num_pipe_per_mec = 4;
1157                 adev->gfx.mec.num_queue_per_pipe = 8;
1158                 break;
1159         }
1160
1161         /* recalculate compute rings to use based on hardware configuration */
1162         num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
1163                              adev->gfx.mec.num_queue_per_pipe) / 2;
1164         adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
1165                                           num_compute_rings);
1166
1167         /* EOP Event */
1168         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1169                               GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1170                               &adev->gfx.eop_irq);
1171         if (r)
1172                 return r;
1173
1174         /* Privileged reg */
1175         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1176                               GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1177                               &adev->gfx.priv_reg_irq);
1178         if (r)
1179                 return r;
1180
1181         /* Privileged inst */
1182         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1183                               GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1184                               &adev->gfx.priv_inst_irq);
1185         if (r)
1186                 return r;
1187
1188         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1189
1190         gfx_v12_0_me_init(adev);
1191
1192         r = gfx_v12_0_rlc_init(adev);
1193         if (r) {
1194                 dev_err(adev->dev, "Failed to init rlc BOs!\n");
1195                 return r;
1196         }
1197
1198         r = gfx_v12_0_mec_init(adev);
1199         if (r) {
1200                 dev_err(adev->dev, "Failed to init MEC BOs!\n");
1201                 return r;
1202         }
1203
1204         /* set up the gfx ring */
1205         for (i = 0; i < adev->gfx.me.num_me; i++) {
1206                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1207                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1208                                 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1209                                         continue;
1210
1211                                 r = gfx_v12_0_gfx_ring_init(adev, ring_id,
1212                                                             i, k, j);
1213                                 if (r)
1214                                         return r;
1215                                 ring_id++;
1216                         }
1217                 }
1218         }
1219
1220         ring_id = 0;
1221         /* set up the compute queues - allocate horizontally across pipes */
1222         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1223                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1224                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1225                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev,
1226                                                                 0, i, k, j))
1227                                         continue;
1228
1229                                 r = gfx_v12_0_compute_ring_init(adev, ring_id,
1230                                                                 i, k, j);
1231                                 if (r)
1232                                         return r;
1233
1234                                 ring_id++;
1235                         }
1236                 }
1237         }
1238
1239         if (!adev->enable_mes_kiq) {
1240                 r = amdgpu_gfx_kiq_init(adev, GFX12_MEC_HPD_SIZE, 0);
1241                 if (r) {
1242                         dev_err(adev->dev, "Failed to init KIQ BOs!\n");
1243                         return r;
1244                 }
1245
1246                 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1247                 if (r)
1248                         return r;
1249         }
1250
1251         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v12_compute_mqd), 0);
1252         if (r)
1253                 return r;
1254
1255         /* allocate visible FB for rlc auto-loading fw */
1256         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1257                 r = gfx_v12_0_rlc_autoload_buffer_init(adev);
1258                 if (r)
1259                         return r;
1260         }
1261
1262         r = gfx_v12_0_gpu_early_init(adev);
1263         if (r)
1264                 return r;
1265
1266         return 0;
1267 }
1268
1269 static void gfx_v12_0_pfp_fini(struct amdgpu_device *adev)
1270 {
1271         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1272                               &adev->gfx.pfp.pfp_fw_gpu_addr,
1273                               (void **)&adev->gfx.pfp.pfp_fw_ptr);
1274
1275         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1276                               &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1277                               (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1278 }
1279
1280 static void gfx_v12_0_me_fini(struct amdgpu_device *adev)
1281 {
1282         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1283                               &adev->gfx.me.me_fw_gpu_addr,
1284                               (void **)&adev->gfx.me.me_fw_ptr);
1285
1286         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1287                                &adev->gfx.me.me_fw_data_gpu_addr,
1288                                (void **)&adev->gfx.me.me_fw_data_ptr);
1289 }
1290
1291 static void gfx_v12_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1292 {
1293         amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1294                         &adev->gfx.rlc.rlc_autoload_gpu_addr,
1295                         (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1296 }
1297
1298 static int gfx_v12_0_sw_fini(void *handle)
1299 {
1300         int i;
1301         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1302
1303         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1304                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1305         for (i = 0; i < adev->gfx.num_compute_rings; i++)
1306                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1307
1308         amdgpu_gfx_mqd_sw_fini(adev, 0);
1309
1310         if (!adev->enable_mes_kiq) {
1311                 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
1312                 amdgpu_gfx_kiq_fini(adev, 0);
1313         }
1314
1315         gfx_v12_0_pfp_fini(adev);
1316         gfx_v12_0_me_fini(adev);
1317         gfx_v12_0_rlc_fini(adev);
1318         gfx_v12_0_mec_fini(adev);
1319
1320         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1321                 gfx_v12_0_rlc_autoload_buffer_fini(adev);
1322
1323         gfx_v12_0_free_microcode(adev);
1324
1325         return 0;
1326 }
1327
1328 static void gfx_v12_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1329                                    u32 sh_num, u32 instance, int xcc_id)
1330 {
1331         u32 data;
1332
1333         if (instance == 0xffffffff)
1334                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1335                                      INSTANCE_BROADCAST_WRITES, 1);
1336         else
1337                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1338                                      instance);
1339
1340         if (se_num == 0xffffffff)
1341                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1342                                      1);
1343         else
1344                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1345
1346         if (sh_num == 0xffffffff)
1347                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1348                                      1);
1349         else
1350                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1351
1352         WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
1353 }
1354
1355 static u32 gfx_v12_0_get_sa_active_bitmap(struct amdgpu_device *adev)
1356 {
1357         u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1358
1359         gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regGRBM_CC_GC_SA_UNIT_DISABLE);
1360         gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1361                                             GRBM_CC_GC_SA_UNIT_DISABLE,
1362                                             SA_DISABLE);
1363         gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGRBM_GC_USER_SA_UNIT_DISABLE);
1364         gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1365                                                  GRBM_GC_USER_SA_UNIT_DISABLE,
1366                                                  SA_DISABLE);
1367         sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1368                                             adev->gfx.config.max_shader_engines);
1369
1370         return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1371 }
1372
1373 static u32 gfx_v12_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1374 {
1375         u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1376         u32 rb_mask;
1377
1378         gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
1379         gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1380                                             CC_RB_BACKEND_DISABLE,
1381                                             BACKEND_DISABLE);
1382         gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
1383         gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
1384                                                  GC_USER_RB_BACKEND_DISABLE,
1385                                                  BACKEND_DISABLE);
1386         rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1387                                             adev->gfx.config.max_shader_engines);
1388
1389         return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
1390 }
1391
1392 static void gfx_v12_0_setup_rb(struct amdgpu_device *adev)
1393 {
1394         u32 rb_bitmap_width_per_sa;
1395         u32 max_sa;
1396         u32 active_sa_bitmap;
1397         u32 global_active_rb_bitmap;
1398         u32 active_rb_bitmap = 0;
1399         u32 i;
1400
1401         /* query sa bitmap from SA_UNIT_DISABLE registers */
1402         active_sa_bitmap = gfx_v12_0_get_sa_active_bitmap(adev);
1403         /* query rb bitmap from RB_BACKEND_DISABLE registers */
1404         global_active_rb_bitmap = gfx_v12_0_get_rb_active_bitmap(adev);
1405
1406         /* generate active rb bitmap according to active sa bitmap */
1407         max_sa = adev->gfx.config.max_shader_engines *
1408                  adev->gfx.config.max_sh_per_se;
1409         rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1410                                  adev->gfx.config.max_sh_per_se;
1411         for (i = 0; i < max_sa; i++) {
1412                 if (active_sa_bitmap & (1 << i))
1413                         active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
1414         }
1415
1416         active_rb_bitmap |= global_active_rb_bitmap;
1417         adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1418         adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1419 }
1420
1421 #define LDS_APP_BASE           0x1
1422 #define SCRATCH_APP_BASE       0x2
1423
1424 static void gfx_v12_0_init_compute_vmid(struct amdgpu_device *adev)
1425 {
1426         int i;
1427         uint32_t sh_mem_bases;
1428         uint32_t data;
1429
1430         /*
1431          * Configure apertures:
1432          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1433          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1434          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1435          */
1436         sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
1437                         SCRATCH_APP_BASE;
1438
1439         mutex_lock(&adev->srbm_mutex);
1440         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1441                 soc24_grbm_select(adev, 0, 0, 0, i);
1442                 /* CP and shaders */
1443                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1444                 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
1445
1446                 /* Enable trap for each kfd vmid. */
1447                 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL);
1448                 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1449                 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data);
1450         }
1451         soc24_grbm_select(adev, 0, 0, 0, 0);
1452         mutex_unlock(&adev->srbm_mutex);
1453 }
1454
1455 static void gfx_v12_0_tcp_harvest(struct amdgpu_device *adev)
1456 {
1457         /* TODO: harvest feature to be added later. */
1458 }
1459
1460 static void gfx_v12_0_get_tcc_info(struct amdgpu_device *adev)
1461 {
1462 }
1463
1464 static void gfx_v12_0_constants_init(struct amdgpu_device *adev)
1465 {
1466         u32 tmp;
1467         int i;
1468
1469         if (!amdgpu_sriov_vf(adev))
1470                 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1471
1472         gfx_v12_0_setup_rb(adev);
1473         gfx_v12_0_get_cu_info(adev, &adev->gfx.cu_info);
1474         gfx_v12_0_get_tcc_info(adev);
1475         adev->gfx.config.pa_sc_tile_steering_override = 0;
1476
1477         /* XXX SH_MEM regs */
1478         /* where to put LDS, scratch, GPUVM in FSA64 space */
1479         mutex_lock(&adev->srbm_mutex);
1480         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1481                 soc24_grbm_select(adev, 0, 0, 0, i);
1482                 /* CP and shaders */
1483                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1484                 if (i != 0) {
1485                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1486                                 (adev->gmc.private_aperture_start >> 48));
1487                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1488                                 (adev->gmc.shared_aperture_start >> 48));
1489                         WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
1490                 }
1491         }
1492         soc24_grbm_select(adev, 0, 0, 0, 0);
1493
1494         mutex_unlock(&adev->srbm_mutex);
1495
1496         gfx_v12_0_init_compute_vmid(adev);
1497 }
1498
1499 static void gfx_v12_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1500                                                 bool enable)
1501 {
1502         u32 tmp;
1503
1504         if (amdgpu_sriov_vf(adev))
1505                 return;
1506
1507         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0);
1508
1509         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
1510                             enable ? 1 : 0);
1511         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
1512                             enable ? 1 : 0);
1513         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
1514                             enable ? 1 : 0);
1515         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
1516                             enable ? 1 : 0);
1517
1518         WREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0, tmp);
1519 }
1520
1521 static int gfx_v12_0_init_csb(struct amdgpu_device *adev)
1522 {
1523         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
1524
1525         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
1526                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
1527         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
1528                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1529         WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
1530
1531         return 0;
1532 }
1533
1534 static void gfx_v12_0_rlc_stop(struct amdgpu_device *adev)
1535 {
1536         u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
1537
1538         tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1539         WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
1540 }
1541
1542 static void gfx_v12_0_rlc_reset(struct amdgpu_device *adev)
1543 {
1544         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1545         udelay(50);
1546         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
1547         udelay(50);
1548 }
1549
1550 static void gfx_v12_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
1551                                              bool enable)
1552 {
1553         uint32_t rlc_pg_cntl;
1554
1555         rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
1556
1557         if (!enable) {
1558                 /* RLC_PG_CNTL[23] = 0 (default)
1559                  * RLC will wait for handshake acks with SMU
1560                  * GFXOFF will be enabled
1561                  * RLC_PG_CNTL[23] = 1
1562                  * RLC will not issue any message to SMU
1563                  * hence no handshake between SMU & RLC
1564                  * GFXOFF will be disabled
1565                  */
1566                 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1567         } else
1568                 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1569         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
1570 }
1571
1572 static void gfx_v12_0_rlc_start(struct amdgpu_device *adev)
1573 {
1574         /* TODO: enable rlc & smu handshake until smu
1575          * and gfxoff feature works as expected */
1576         if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
1577                 gfx_v12_0_rlc_smu_handshake_cntl(adev, false);
1578
1579         WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
1580         udelay(50);
1581 }
1582
1583 static void gfx_v12_0_rlc_enable_srm(struct amdgpu_device *adev)
1584 {
1585         uint32_t tmp;
1586
1587         /* enable Save Restore Machine */
1588         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
1589         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
1590         tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
1591         WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
1592 }
1593
1594 static void gfx_v12_0_load_rlcg_microcode(struct amdgpu_device *adev)
1595 {
1596         const struct rlc_firmware_header_v2_0 *hdr;
1597         const __le32 *fw_data;
1598         unsigned i, fw_size;
1599
1600         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1601         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1602                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1603         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1604
1605         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
1606                      RLCG_UCODE_LOADING_START_ADDRESS);
1607
1608         for (i = 0; i < fw_size; i++)
1609                 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
1610                              le32_to_cpup(fw_data++));
1611
1612         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
1613 }
1614
1615 static void gfx_v12_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
1616 {
1617         const struct rlc_firmware_header_v2_2 *hdr;
1618         const __le32 *fw_data;
1619         unsigned i, fw_size;
1620         u32 tmp;
1621
1622         hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1623
1624         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1625                         le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
1626         fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
1627
1628         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
1629
1630         for (i = 0; i < fw_size; i++) {
1631                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1632                         msleep(1);
1633                 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
1634                                 le32_to_cpup(fw_data++));
1635         }
1636
1637         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1638
1639         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1640                         le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
1641         fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
1642
1643         WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
1644         for (i = 0; i < fw_size; i++) {
1645                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1646                         msleep(1);
1647                 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
1648                                 le32_to_cpup(fw_data++));
1649         }
1650
1651         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1652
1653         tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
1654         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
1655         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
1656         WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
1657 }
1658
1659 static int gfx_v12_0_rlc_load_microcode(struct amdgpu_device *adev)
1660 {
1661         const struct rlc_firmware_header_v2_0 *hdr;
1662         uint16_t version_major;
1663         uint16_t version_minor;
1664
1665         if (!adev->gfx.rlc_fw)
1666                 return -EINVAL;
1667
1668         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1669         amdgpu_ucode_print_rlc_hdr(&hdr->header);
1670
1671         version_major = le16_to_cpu(hdr->header.header_version_major);
1672         version_minor = le16_to_cpu(hdr->header.header_version_minor);
1673
1674         if (version_major == 2) {
1675                 gfx_v12_0_load_rlcg_microcode(adev);
1676                 if (amdgpu_dpm == 1) {
1677                         if (version_minor >= 2)
1678                                 gfx_v12_0_load_rlc_iram_dram_microcode(adev);
1679                 }
1680
1681                 return 0;
1682         }
1683
1684         return -EINVAL;
1685 }
1686
1687 static int gfx_v12_0_rlc_resume(struct amdgpu_device *adev)
1688 {
1689         int r;
1690
1691         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1692                 gfx_v12_0_init_csb(adev);
1693
1694                 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
1695                         gfx_v12_0_rlc_enable_srm(adev);
1696         } else {
1697                 if (amdgpu_sriov_vf(adev)) {
1698                         gfx_v12_0_init_csb(adev);
1699                         return 0;
1700                 }
1701
1702                 adev->gfx.rlc.funcs->stop(adev);
1703
1704                 /* disable CG */
1705                 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
1706
1707                 /* disable PG */
1708                 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
1709
1710                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1711                         /* legacy rlc firmware loading */
1712                         r = gfx_v12_0_rlc_load_microcode(adev);
1713                         if (r)
1714                                 return r;
1715                 }
1716
1717                 gfx_v12_0_init_csb(adev);
1718
1719                 adev->gfx.rlc.funcs->start(adev);
1720         }
1721
1722         return 0;
1723 }
1724
1725 static void gfx_v12_0_config_gfx_rs64(struct amdgpu_device *adev)
1726 {
1727         const struct gfx_firmware_header_v2_0 *pfp_hdr;
1728         const struct gfx_firmware_header_v2_0 *me_hdr;
1729         const struct gfx_firmware_header_v2_0 *mec_hdr;
1730         uint32_t pipe_id, tmp;
1731
1732         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
1733                 adev->gfx.mec_fw->data;
1734         me_hdr = (const struct gfx_firmware_header_v2_0 *)
1735                 adev->gfx.me_fw->data;
1736         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
1737                 adev->gfx.pfp_fw->data;
1738
1739         /* config pfp program start addr */
1740         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
1741                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
1742                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
1743                         (pfp_hdr->ucode_start_addr_hi << 30) |
1744                         (pfp_hdr->ucode_start_addr_lo >> 2));
1745                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
1746                         pfp_hdr->ucode_start_addr_hi >> 2);
1747         }
1748         soc24_grbm_select(adev, 0, 0, 0, 0);
1749
1750         /* reset pfp pipe */
1751         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
1752         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
1753         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
1754         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1755
1756         /* clear pfp pipe reset */
1757         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
1758         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
1759         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1760
1761         /* config me program start addr */
1762         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
1763                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
1764                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
1765                         (me_hdr->ucode_start_addr_hi << 30) |
1766                         (me_hdr->ucode_start_addr_lo >> 2));
1767                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
1768                         me_hdr->ucode_start_addr_hi>>2);
1769         }
1770         soc24_grbm_select(adev, 0, 0, 0, 0);
1771
1772         /* reset me pipe */
1773         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
1774         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
1775         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
1776         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1777
1778         /* clear me pipe reset */
1779         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
1780         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
1781         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1782
1783         /* config mec program start addr */
1784         for (pipe_id = 0; pipe_id < 4; pipe_id++) {
1785                 soc24_grbm_select(adev, 1, pipe_id, 0, 0);
1786                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
1787                                         mec_hdr->ucode_start_addr_lo >> 2 |
1788                                         mec_hdr->ucode_start_addr_hi << 30);
1789                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
1790                                         mec_hdr->ucode_start_addr_hi >> 2);
1791         }
1792         soc24_grbm_select(adev, 0, 0, 0, 0);
1793
1794         /* reset mec pipe */
1795         tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
1796         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
1797         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
1798         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
1799         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
1800         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
1801
1802         /* clear mec pipe reset */
1803         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
1804         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
1805         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
1806         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
1807         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
1808 }
1809
1810 static void gfx_v12_0_set_pfp_ucode_start_addr(struct amdgpu_device *adev)
1811 {
1812         const struct gfx_firmware_header_v2_0 *cp_hdr;
1813         unsigned pipe_id, tmp;
1814
1815         cp_hdr = (const struct gfx_firmware_header_v2_0 *)
1816                 adev->gfx.pfp_fw->data;
1817         mutex_lock(&adev->srbm_mutex);
1818         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
1819                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
1820                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
1821                              (cp_hdr->ucode_start_addr_hi << 30) |
1822                              (cp_hdr->ucode_start_addr_lo >> 2));
1823                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
1824                              cp_hdr->ucode_start_addr_hi>>2);
1825
1826                 /*
1827                  * Program CP_ME_CNTL to reset given PIPE to take
1828                  * effect of CP_PFP_PRGRM_CNTR_START.
1829                  */
1830                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
1831                 if (pipe_id == 0)
1832                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
1833                                         PFP_PIPE0_RESET, 1);
1834                 else
1835                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
1836                                         PFP_PIPE1_RESET, 1);
1837                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1838
1839                 /* Clear pfp pipe0 reset bit. */
1840                 if (pipe_id == 0)
1841                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
1842                                         PFP_PIPE0_RESET, 0);
1843                 else
1844                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
1845                                         PFP_PIPE1_RESET, 0);
1846                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1847         }
1848         soc24_grbm_select(adev, 0, 0, 0, 0);
1849         mutex_unlock(&adev->srbm_mutex);
1850 }
1851
1852 static void gfx_v12_0_set_me_ucode_start_addr(struct amdgpu_device *adev)
1853 {
1854         const struct gfx_firmware_header_v2_0 *cp_hdr;
1855         unsigned pipe_id, tmp;
1856
1857         cp_hdr = (const struct gfx_firmware_header_v2_0 *)
1858                 adev->gfx.me_fw->data;
1859         mutex_lock(&adev->srbm_mutex);
1860         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
1861                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
1862                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
1863                              (cp_hdr->ucode_start_addr_hi << 30) |
1864                              (cp_hdr->ucode_start_addr_lo >> 2) );
1865                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
1866                              cp_hdr->ucode_start_addr_hi>>2);
1867
1868                 /*
1869                  * Program CP_ME_CNTL to reset given PIPE to take
1870                  * effect of CP_ME_PRGRM_CNTR_START.
1871                  */
1872                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
1873                 if (pipe_id == 0)
1874                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
1875                                         ME_PIPE0_RESET, 1);
1876                 else
1877                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
1878                                         ME_PIPE1_RESET, 1);
1879                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1880
1881                 /* Clear pfp pipe0 reset bit. */
1882                 if (pipe_id == 0)
1883                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
1884                                         ME_PIPE0_RESET, 0);
1885                 else
1886                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
1887                                         ME_PIPE1_RESET, 0);
1888                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1889         }
1890         soc24_grbm_select(adev, 0, 0, 0, 0);
1891         mutex_unlock(&adev->srbm_mutex);
1892 }
1893
1894 static void gfx_v12_0_set_mec_ucode_start_addr(struct amdgpu_device *adev)
1895 {
1896         const struct gfx_firmware_header_v2_0 *cp_hdr;
1897         unsigned pipe_id;
1898
1899         cp_hdr = (const struct gfx_firmware_header_v2_0 *)
1900                 adev->gfx.mec_fw->data;
1901         mutex_lock(&adev->srbm_mutex);
1902         for (pipe_id = 0; pipe_id < adev->gfx.mec.num_pipe_per_mec; pipe_id++) {
1903                 soc24_grbm_select(adev, 1, pipe_id, 0, 0);
1904                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
1905                              cp_hdr->ucode_start_addr_lo >> 2 |
1906                              cp_hdr->ucode_start_addr_hi << 30);
1907                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
1908                              cp_hdr->ucode_start_addr_hi >> 2);
1909         }
1910         soc24_grbm_select(adev, 0, 0, 0, 0);
1911         mutex_unlock(&adev->srbm_mutex);
1912 }
1913
1914 static int gfx_v12_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
1915 {
1916         uint32_t cp_status;
1917         uint32_t bootload_status;
1918         int i;
1919
1920         for (i = 0; i < adev->usec_timeout; i++) {
1921                 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
1922                 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
1923
1924                 if ((cp_status == 0) &&
1925                     (REG_GET_FIELD(bootload_status,
1926                         RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
1927                         break;
1928                 }
1929                 udelay(1);
1930                 if (amdgpu_emu_mode)
1931                         msleep(10);
1932         }
1933
1934         if (i >= adev->usec_timeout) {
1935                 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
1936                 return -ETIMEDOUT;
1937         }
1938
1939         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1940                 gfx_v12_0_set_pfp_ucode_start_addr(adev);
1941                 gfx_v12_0_set_me_ucode_start_addr(adev);
1942                 gfx_v12_0_set_mec_ucode_start_addr(adev);
1943         }
1944
1945         return 0;
1946 }
1947
1948 static int gfx_v12_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
1949 {
1950         int i;
1951         u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
1952
1953         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
1954         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
1955         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1956
1957         for (i = 0; i < adev->usec_timeout; i++) {
1958                 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
1959                         break;
1960                 udelay(1);
1961         }
1962
1963         if (i >= adev->usec_timeout)
1964                 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
1965
1966         return 0;
1967 }
1968
1969 static int gfx_v12_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
1970 {
1971         int r;
1972         const struct gfx_firmware_header_v2_0 *pfp_hdr;
1973         const __le32 *fw_ucode, *fw_data;
1974         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
1975         uint32_t tmp;
1976         uint32_t usec_timeout = 50000;  /* wait for 50ms */
1977
1978         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
1979                 adev->gfx.pfp_fw->data;
1980
1981         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
1982
1983         /* instruction */
1984         fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
1985                 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
1986         fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
1987         /* data */
1988         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1989                 le32_to_cpu(pfp_hdr->data_offset_bytes));
1990         fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
1991
1992         /* 64kb align */
1993         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
1994                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
1995                                       &adev->gfx.pfp.pfp_fw_obj,
1996                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
1997                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
1998         if (r) {
1999                 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
2000                 gfx_v12_0_pfp_fini(adev);
2001                 return r;
2002         }
2003
2004         r = amdgpu_bo_create_reserved(adev, fw_data_size,
2005                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2006                                       &adev->gfx.pfp.pfp_fw_data_obj,
2007                                       &adev->gfx.pfp.pfp_fw_data_gpu_addr,
2008                                       (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
2009         if (r) {
2010                 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
2011                 gfx_v12_0_pfp_fini(adev);
2012                 return r;
2013         }
2014
2015         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
2016         memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
2017
2018         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2019         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
2020         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2021         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
2022
2023         if (amdgpu_emu_mode == 1)
2024                 adev->hdp.funcs->flush_hdp(adev, NULL);
2025
2026         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2027                 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2028         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2029                 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2030
2031         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2032         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2033         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2034         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2035         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2036
2037         /*
2038          * Programming any of the CP_PFP_IC_BASE registers
2039          * forces invalidation of the ME L1 I$. Wait for the
2040          * invalidation complete
2041          */
2042         for (i = 0; i < usec_timeout; i++) {
2043                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2044                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2045                         INVALIDATE_CACHE_COMPLETE))
2046                         break;
2047                 udelay(1);
2048         }
2049
2050         if (i >= usec_timeout) {
2051                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2052                 return -EINVAL;
2053         }
2054
2055         /* Prime the L1 instruction caches */
2056         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2057         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2058         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2059         /* Waiting for cache primed*/
2060         for (i = 0; i < usec_timeout; i++) {
2061                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2062                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2063                         ICACHE_PRIMED))
2064                         break;
2065                 udelay(1);
2066         }
2067
2068         if (i >= usec_timeout) {
2069                 dev_err(adev->dev, "failed to prime instruction cache\n");
2070                 return -EINVAL;
2071         }
2072
2073         mutex_lock(&adev->srbm_mutex);
2074         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2075                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2076
2077                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2078                         lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
2079                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2080                         upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
2081         }
2082         soc24_grbm_select(adev, 0, 0, 0, 0);
2083         mutex_unlock(&adev->srbm_mutex);
2084
2085         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2086         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2087         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2088         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2089
2090         /* Invalidate the data caches */
2091         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2092         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2093         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2094
2095         for (i = 0; i < usec_timeout; i++) {
2096                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2097                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2098                         INVALIDATE_DCACHE_COMPLETE))
2099                         break;
2100                 udelay(1);
2101         }
2102
2103         if (i >= usec_timeout) {
2104                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2105                 return -EINVAL;
2106         }
2107
2108         gfx_v12_0_set_pfp_ucode_start_addr(adev);
2109
2110         return 0;
2111 }
2112
2113 static int gfx_v12_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
2114 {
2115         int r;
2116         const struct gfx_firmware_header_v2_0 *me_hdr;
2117         const __le32 *fw_ucode, *fw_data;
2118         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
2119         uint32_t tmp;
2120         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2121
2122         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2123                 adev->gfx.me_fw->data;
2124
2125         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2126
2127         /* instruction */
2128         fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
2129                 le32_to_cpu(me_hdr->ucode_offset_bytes));
2130         fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
2131         /* data */
2132         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
2133                 le32_to_cpu(me_hdr->data_offset_bytes));
2134         fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
2135
2136         /* 64kb align*/
2137         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2138                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2139                                       &adev->gfx.me.me_fw_obj,
2140                                       &adev->gfx.me.me_fw_gpu_addr,
2141                                       (void **)&adev->gfx.me.me_fw_ptr);
2142         if (r) {
2143                 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
2144                 gfx_v12_0_me_fini(adev);
2145                 return r;
2146         }
2147
2148         r = amdgpu_bo_create_reserved(adev, fw_data_size,
2149                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2150                                       &adev->gfx.me.me_fw_data_obj,
2151                                       &adev->gfx.me.me_fw_data_gpu_addr,
2152                                       (void **)&adev->gfx.me.me_fw_data_ptr);
2153         if (r) {
2154                 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
2155                 gfx_v12_0_pfp_fini(adev);
2156                 return r;
2157         }
2158
2159         memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
2160         memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
2161
2162         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
2163         amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
2164         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
2165         amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
2166
2167         if (amdgpu_emu_mode == 1)
2168                 adev->hdp.funcs->flush_hdp(adev, NULL);
2169
2170         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2171                 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
2172         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2173                 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
2174
2175         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2176         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2177         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2178         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2179         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2180
2181         /*
2182          * Programming any of the CP_ME_IC_BASE registers
2183          * forces invalidation of the ME L1 I$. Wait for the
2184          * invalidation complete
2185          */
2186         for (i = 0; i < usec_timeout; i++) {
2187                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2188                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2189                         INVALIDATE_CACHE_COMPLETE))
2190                         break;
2191                 udelay(1);
2192         }
2193
2194         if (i >= usec_timeout) {
2195                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2196                 return -EINVAL;
2197         }
2198
2199         /* Prime the instruction caches */
2200         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2201         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2202         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2203
2204         /* Waiting for instruction cache primed*/
2205         for (i = 0; i < usec_timeout; i++) {
2206                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2207                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2208                         ICACHE_PRIMED))
2209                         break;
2210                 udelay(1);
2211         }
2212
2213         if (i >= usec_timeout) {
2214                 dev_err(adev->dev, "failed to prime instruction cache\n");
2215                 return -EINVAL;
2216         }
2217
2218         mutex_lock(&adev->srbm_mutex);
2219         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2220                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2221
2222                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2223                         lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
2224                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2225                         upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
2226         }
2227         soc24_grbm_select(adev, 0, 0, 0, 0);
2228         mutex_unlock(&adev->srbm_mutex);
2229
2230         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2231         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2232         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2233         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2234
2235         /* Invalidate the data caches */
2236         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2237         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2238         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2239
2240         for (i = 0; i < usec_timeout; i++) {
2241                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2242                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2243                         INVALIDATE_DCACHE_COMPLETE))
2244                         break;
2245                 udelay(1);
2246         }
2247
2248         if (i >= usec_timeout) {
2249                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2250                 return -EINVAL;
2251         }
2252
2253         gfx_v12_0_set_me_ucode_start_addr(adev);
2254
2255         return 0;
2256 }
2257
2258 static int gfx_v12_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2259 {
2260         int r;
2261
2262         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
2263                 return -EINVAL;
2264
2265         gfx_v12_0_cp_gfx_enable(adev, false);
2266
2267         r = gfx_v12_0_cp_gfx_load_pfp_microcode_rs64(adev);
2268         if (r) {
2269                 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
2270                 return r;
2271         }
2272
2273         r = gfx_v12_0_cp_gfx_load_me_microcode_rs64(adev);
2274         if (r) {
2275                 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
2276                 return r;
2277         }
2278
2279         return 0;
2280 }
2281
2282 static int gfx_v12_0_cp_gfx_start(struct amdgpu_device *adev)
2283 {
2284         /* init the CP */
2285         WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
2286                      adev->gfx.config.max_hw_contexts - 1);
2287         WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
2288
2289         if (!amdgpu_async_gfx_ring)
2290                 gfx_v12_0_cp_gfx_enable(adev, true);
2291
2292         return 0;
2293 }
2294
2295 static void gfx_v12_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
2296                                          CP_PIPE_ID pipe)
2297 {
2298         u32 tmp;
2299
2300         tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
2301         tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
2302
2303         WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
2304 }
2305
2306 static void gfx_v12_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
2307                                           struct amdgpu_ring *ring)
2308 {
2309         u32 tmp;
2310
2311         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
2312         if (ring->use_doorbell) {
2313                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2314                                     DOORBELL_OFFSET, ring->doorbell_index);
2315                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2316                                     DOORBELL_EN, 1);
2317         } else {
2318                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2319                                     DOORBELL_EN, 0);
2320         }
2321         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
2322
2323         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
2324                             DOORBELL_RANGE_LOWER, ring->doorbell_index);
2325         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
2326
2327         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
2328                      CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
2329 }
2330
2331 static int gfx_v12_0_cp_gfx_resume(struct amdgpu_device *adev)
2332 {
2333         struct amdgpu_ring *ring;
2334         u32 tmp;
2335         u32 rb_bufsz;
2336         u64 rb_addr, rptr_addr, wptr_gpu_addr;
2337         u32 i;
2338
2339         /* Set the write pointer delay */
2340         WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
2341
2342         /* set the RB to use vmid 0 */
2343         WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
2344
2345         /* Init gfx ring 0 for pipe 0 */
2346         mutex_lock(&adev->srbm_mutex);
2347         gfx_v12_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
2348
2349         /* Set ring buffer size */
2350         ring = &adev->gfx.gfx_ring[0];
2351         rb_bufsz = order_base_2(ring->ring_size / 8);
2352         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
2353         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
2354         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
2355
2356         /* Initialize the ring buffer's write pointers */
2357         ring->wptr = 0;
2358         WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
2359         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
2360
2361         /* set the wb address wether it's enabled or not */
2362         rptr_addr = ring->rptr_gpu_addr;
2363         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
2364         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
2365                      CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
2366
2367         wptr_gpu_addr = ring->wptr_gpu_addr;
2368         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
2369                      lower_32_bits(wptr_gpu_addr));
2370         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
2371                      upper_32_bits(wptr_gpu_addr));
2372
2373         mdelay(1);
2374         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
2375
2376         rb_addr = ring->gpu_addr >> 8;
2377         WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
2378         WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
2379
2380         WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
2381
2382         gfx_v12_0_cp_gfx_set_doorbell(adev, ring);
2383         mutex_unlock(&adev->srbm_mutex);
2384
2385         /* Switch to pipe 0 */
2386         mutex_lock(&adev->srbm_mutex);
2387         gfx_v12_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
2388         mutex_unlock(&adev->srbm_mutex);
2389
2390         /* start the ring */
2391         gfx_v12_0_cp_gfx_start(adev);
2392
2393         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2394                 ring = &adev->gfx.gfx_ring[i];
2395                 ring->sched.ready = true;
2396         }
2397
2398         return 0;
2399 }
2400
2401 static void gfx_v12_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
2402 {
2403         u32 data;
2404
2405         data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
2406         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
2407                                                  enable ? 0 : 1);
2408         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
2409                                                  enable ? 0 : 1);
2410         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
2411                                                  enable ? 0 : 1);
2412         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
2413                                                  enable ? 0 : 1);
2414         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
2415                                                  enable ? 0 : 1);
2416         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
2417                                                  enable ? 1 : 0);
2418         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
2419                                                  enable ? 1 : 0);
2420         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
2421                                                  enable ? 1 : 0);
2422         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
2423                                                  enable ? 1 : 0);
2424         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
2425                                                  enable ? 0 : 1);
2426         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
2427
2428         adev->gfx.kiq[0].ring.sched.ready = enable;
2429
2430         udelay(50);
2431 }
2432
2433 static int gfx_v12_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
2434 {
2435         const struct gfx_firmware_header_v2_0 *mec_hdr;
2436         const __le32 *fw_ucode, *fw_data;
2437         u32 tmp, fw_ucode_size, fw_data_size;
2438         u32 i, usec_timeout = 50000; /* Wait for 50 ms */
2439         u32 *fw_ucode_ptr, *fw_data_ptr;
2440         int r;
2441
2442         if (!adev->gfx.mec_fw)
2443                 return -EINVAL;
2444
2445         gfx_v12_0_cp_compute_enable(adev, false);
2446
2447         mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
2448         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
2449
2450         fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
2451                                 le32_to_cpu(mec_hdr->ucode_offset_bytes));
2452         fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
2453
2454         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
2455                                 le32_to_cpu(mec_hdr->data_offset_bytes));
2456         fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
2457
2458         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2459                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2460                                       &adev->gfx.mec.mec_fw_obj,
2461                                       &adev->gfx.mec.mec_fw_gpu_addr,
2462                                       (void **)&fw_ucode_ptr);
2463         if (r) {
2464                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
2465                 gfx_v12_0_mec_fini(adev);
2466                 return r;
2467         }
2468
2469         r = amdgpu_bo_create_reserved(adev,
2470                                       ALIGN(fw_data_size, 64 * 1024) *
2471                                       adev->gfx.mec.num_pipe_per_mec,
2472                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2473                                       &adev->gfx.mec.mec_fw_data_obj,
2474                                       &adev->gfx.mec.mec_fw_data_gpu_addr,
2475                                       (void **)&fw_data_ptr);
2476         if (r) {
2477                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
2478                 gfx_v12_0_mec_fini(adev);
2479                 return r;
2480         }
2481
2482         memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
2483         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2484                 memcpy(fw_data_ptr + i * ALIGN(fw_data_size, 64 * 1024) / 4, fw_data, fw_data_size);
2485         }
2486
2487         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
2488         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
2489         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
2490         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
2491
2492         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2493         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2494         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2495         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2496         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2497
2498         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2499         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2500         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2501         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2502
2503         mutex_lock(&adev->srbm_mutex);
2504         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2505                 soc24_grbm_select(adev, 1, i, 0, 0);
2506
2507                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO,
2508                              lower_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
2509                                            i * ALIGN(fw_data_size, 64 * 1024)));
2510                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2511                              upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
2512                                            i * ALIGN(fw_data_size, 64 * 1024)));
2513
2514                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2515                              lower_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
2516                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2517                              upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
2518         }
2519         mutex_unlock(&adev->srbm_mutex);
2520         soc24_grbm_select(adev, 0, 0, 0, 0);
2521
2522         /* Trigger an invalidation of the L1 instruction caches */
2523         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2524         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2525         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2526
2527         /* Wait for invalidation complete */
2528         for (i = 0; i < usec_timeout; i++) {
2529                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2530                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2531                                        INVALIDATE_DCACHE_COMPLETE))
2532                         break;
2533                 udelay(1);
2534         }
2535
2536         if (i >= usec_timeout) {
2537                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2538                 return -EINVAL;
2539         }
2540
2541         /* Trigger an invalidation of the L1 instruction caches */
2542         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2543         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2544         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2545
2546         /* Wait for invalidation complete */
2547         for (i = 0; i < usec_timeout; i++) {
2548                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2549                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2550                                        INVALIDATE_CACHE_COMPLETE))
2551                         break;
2552                 udelay(1);
2553         }
2554
2555         if (i >= usec_timeout) {
2556                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2557                 return -EINVAL;
2558         }
2559
2560         gfx_v12_0_set_mec_ucode_start_addr(adev);
2561
2562         return 0;
2563 }
2564
2565 static void gfx_v12_0_kiq_setting(struct amdgpu_ring *ring)
2566 {
2567         uint32_t tmp;
2568         struct amdgpu_device *adev = ring->adev;
2569
2570         /* tell RLC which is KIQ queue */
2571         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
2572         tmp &= 0xffffff00;
2573         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
2574         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
2575         tmp |= 0x80;
2576         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
2577 }
2578
2579 static void gfx_v12_0_cp_set_doorbell_range(struct amdgpu_device *adev)
2580 {
2581         /* set graphics engine doorbell range */
2582         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
2583                      (adev->doorbell_index.gfx_ring0 * 2) << 2);
2584         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
2585                      (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
2586
2587         /* set compute engine doorbell range */
2588         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
2589                      (adev->doorbell_index.kiq * 2) << 2);
2590         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
2591                      (adev->doorbell_index.userqueue_end * 2) << 2);
2592 }
2593
2594 static int gfx_v12_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
2595                                   struct amdgpu_mqd_prop *prop)
2596 {
2597         struct v12_gfx_mqd *mqd = m;
2598         uint64_t hqd_gpu_addr, wb_gpu_addr;
2599         uint32_t tmp;
2600         uint32_t rb_bufsz;
2601
2602         /* set up gfx hqd wptr */
2603         mqd->cp_gfx_hqd_wptr = 0;
2604         mqd->cp_gfx_hqd_wptr_hi = 0;
2605
2606         /* set the pointer to the MQD */
2607         mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
2608         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
2609
2610         /* set up mqd control */
2611         tmp = RREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL);
2612         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
2613         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
2614         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
2615         mqd->cp_gfx_mqd_control = tmp;
2616
2617         /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
2618         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID);
2619         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
2620         mqd->cp_gfx_hqd_vmid = 0;
2621
2622         /* set up default queue priority level
2623          * 0x0 = low priority, 0x1 = high priority */
2624         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY);
2625         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, 0);
2626         mqd->cp_gfx_hqd_queue_priority = tmp;
2627
2628         /* set up time quantum */
2629         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM);
2630         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
2631         mqd->cp_gfx_hqd_quantum = tmp;
2632
2633         /* set up gfx hqd base. this is similar as CP_RB_BASE */
2634         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
2635         mqd->cp_gfx_hqd_base = hqd_gpu_addr;
2636         mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
2637
2638         /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
2639         wb_gpu_addr = prop->rptr_gpu_addr;
2640         mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
2641         mqd->cp_gfx_hqd_rptr_addr_hi =
2642                 upper_32_bits(wb_gpu_addr) & 0xffff;
2643
2644         /* set up rb_wptr_poll addr */
2645         wb_gpu_addr = prop->wptr_gpu_addr;
2646         mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
2647         mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
2648
2649         /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
2650         rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
2651         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL);
2652         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
2653         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
2654 #ifdef __BIG_ENDIAN
2655         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
2656 #endif
2657         mqd->cp_gfx_hqd_cntl = tmp;
2658
2659         /* set up cp_doorbell_control */
2660         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
2661         if (prop->use_doorbell) {
2662                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2663                                     DOORBELL_OFFSET, prop->doorbell_index);
2664                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2665                                     DOORBELL_EN, 1);
2666         } else
2667                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2668                                     DOORBELL_EN, 0);
2669         mqd->cp_rb_doorbell_control = tmp;
2670
2671         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2672         mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR);
2673
2674         /* active the queue */
2675         mqd->cp_gfx_hqd_active = 1;
2676
2677         return 0;
2678 }
2679
2680 static int gfx_v12_0_gfx_init_queue(struct amdgpu_ring *ring)
2681 {
2682         struct amdgpu_device *adev = ring->adev;
2683         struct v12_gfx_mqd *mqd = ring->mqd_ptr;
2684         int mqd_idx = ring - &adev->gfx.gfx_ring[0];
2685
2686         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
2687                 memset((void *)mqd, 0, sizeof(*mqd));
2688                 mutex_lock(&adev->srbm_mutex);
2689                 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
2690                 amdgpu_ring_init_mqd(ring);
2691                 soc24_grbm_select(adev, 0, 0, 0, 0);
2692                 mutex_unlock(&adev->srbm_mutex);
2693                 if (adev->gfx.me.mqd_backup[mqd_idx])
2694                         memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
2695         } else {
2696                 /* restore mqd with the backup copy */
2697                 if (adev->gfx.me.mqd_backup[mqd_idx])
2698                         memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
2699                 /* reset the ring */
2700                 ring->wptr = 0;
2701                 *ring->wptr_cpu_addr = 0;
2702                 amdgpu_ring_clear_ring(ring);
2703         }
2704
2705         return 0;
2706 }
2707
2708 static int gfx_v12_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
2709 {
2710         int r, i;
2711         struct amdgpu_ring *ring;
2712
2713         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2714                 ring = &adev->gfx.gfx_ring[i];
2715
2716                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
2717                 if (unlikely(r != 0))
2718                         goto done;
2719
2720                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
2721                 if (!r) {
2722                         r = gfx_v12_0_gfx_init_queue(ring);
2723                         amdgpu_bo_kunmap(ring->mqd_obj);
2724                         ring->mqd_ptr = NULL;
2725                 }
2726                 amdgpu_bo_unreserve(ring->mqd_obj);
2727                 if (r)
2728                         goto done;
2729         }
2730
2731         r = amdgpu_gfx_enable_kgq(adev, 0);
2732         if (r)
2733                 goto done;
2734
2735         r = gfx_v12_0_cp_gfx_start(adev);
2736         if (r)
2737                 goto done;
2738
2739         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2740                 ring = &adev->gfx.gfx_ring[i];
2741                 ring->sched.ready = true;
2742         }
2743 done:
2744         return r;
2745 }
2746
2747 static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
2748                                       struct amdgpu_mqd_prop *prop)
2749 {
2750         struct v12_compute_mqd *mqd = m;
2751         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
2752         uint32_t tmp;
2753
2754         mqd->header = 0xC0310800;
2755         mqd->compute_pipelinestat_enable = 0x00000001;
2756         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
2757         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
2758         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
2759         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
2760         mqd->compute_misc_reserved = 0x00000007;
2761
2762         eop_base_addr = prop->eop_gpu_addr >> 8;
2763         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
2764         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
2765
2766         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2767         tmp = RREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL);
2768         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
2769                         (order_base_2(GFX12_MEC_HPD_SIZE / 4) - 1));
2770
2771         mqd->cp_hqd_eop_control = tmp;
2772
2773         /* enable doorbell? */
2774         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
2775
2776         if (prop->use_doorbell) {
2777                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2778                                     DOORBELL_OFFSET, prop->doorbell_index);
2779                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2780                                     DOORBELL_EN, 1);
2781                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2782                                     DOORBELL_SOURCE, 0);
2783                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2784                                     DOORBELL_HIT, 0);
2785         } else {
2786                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2787                                     DOORBELL_EN, 0);
2788         }
2789
2790         mqd->cp_hqd_pq_doorbell_control = tmp;
2791
2792         /* disable the queue if it's active */
2793         mqd->cp_hqd_dequeue_request = 0;
2794         mqd->cp_hqd_pq_rptr = 0;
2795         mqd->cp_hqd_pq_wptr_lo = 0;
2796         mqd->cp_hqd_pq_wptr_hi = 0;
2797
2798         /* set the pointer to the MQD */
2799         mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
2800         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
2801
2802         /* set MQD vmid to 0 */
2803         tmp = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
2804         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
2805         mqd->cp_mqd_control = tmp;
2806
2807         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2808         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
2809         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
2810         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
2811
2812         /* set up the HQD, this is similar to CP_RB0_CNTL */
2813         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL);
2814         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
2815                             (order_base_2(prop->queue_size / 4) - 1));
2816         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
2817                             (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
2818         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
2819         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
2820         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
2821         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
2822         mqd->cp_hqd_pq_control = tmp;
2823
2824         /* set the wb address whether it's enabled or not */
2825         wb_gpu_addr = prop->rptr_gpu_addr;
2826         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
2827         mqd->cp_hqd_pq_rptr_report_addr_hi =
2828                 upper_32_bits(wb_gpu_addr) & 0xffff;
2829
2830         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2831         wb_gpu_addr = prop->wptr_gpu_addr;
2832         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
2833         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
2834
2835         tmp = 0;
2836         /* enable the doorbell if requested */
2837         if (prop->use_doorbell) {
2838                 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
2839                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2840                                 DOORBELL_OFFSET, prop->doorbell_index);
2841
2842                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2843                                     DOORBELL_EN, 1);
2844                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2845                                     DOORBELL_SOURCE, 0);
2846                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2847                                     DOORBELL_HIT, 0);
2848         }
2849
2850         mqd->cp_hqd_pq_doorbell_control = tmp;
2851
2852         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2853         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR);
2854
2855         /* set the vmid for the queue */
2856         mqd->cp_hqd_vmid = 0;
2857
2858         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE);
2859         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
2860         mqd->cp_hqd_persistent_state = tmp;
2861
2862         /* set MIN_IB_AVAIL_SIZE */
2863         tmp = RREG32_SOC15(GC, 0, regCP_HQD_IB_CONTROL);
2864         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
2865         mqd->cp_hqd_ib_control = tmp;
2866
2867         /* set static priority for a compute queue/ring */
2868         mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
2869         mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
2870
2871         mqd->cp_hqd_active = prop->hqd_active;
2872
2873         return 0;
2874 }
2875
2876 static int gfx_v12_0_kiq_init_register(struct amdgpu_ring *ring)
2877 {
2878         struct amdgpu_device *adev = ring->adev;
2879         struct v12_compute_mqd *mqd = ring->mqd_ptr;
2880         int j;
2881
2882         /* inactivate the queue */
2883         if (amdgpu_sriov_vf(adev))
2884                 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
2885
2886         /* disable wptr polling */
2887         WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
2888
2889         /* write the EOP addr */
2890         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
2891                mqd->cp_hqd_eop_base_addr_lo);
2892         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
2893                mqd->cp_hqd_eop_base_addr_hi);
2894
2895         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2896         WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
2897                mqd->cp_hqd_eop_control);
2898
2899         /* enable doorbell? */
2900         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
2901                mqd->cp_hqd_pq_doorbell_control);
2902
2903         /* disable the queue if it's active */
2904         if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
2905                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
2906                 for (j = 0; j < adev->usec_timeout; j++) {
2907                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
2908                                 break;
2909                         udelay(1);
2910                 }
2911                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
2912                        mqd->cp_hqd_dequeue_request);
2913                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
2914                        mqd->cp_hqd_pq_rptr);
2915                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
2916                        mqd->cp_hqd_pq_wptr_lo);
2917                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
2918                        mqd->cp_hqd_pq_wptr_hi);
2919         }
2920
2921         /* set the pointer to the MQD */
2922         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
2923                mqd->cp_mqd_base_addr_lo);
2924         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
2925                mqd->cp_mqd_base_addr_hi);
2926
2927         /* set MQD vmid to 0 */
2928         WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
2929                mqd->cp_mqd_control);
2930
2931         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2932         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
2933                mqd->cp_hqd_pq_base_lo);
2934         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
2935                mqd->cp_hqd_pq_base_hi);
2936
2937         /* set up the HQD, this is similar to CP_RB0_CNTL */
2938         WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
2939                mqd->cp_hqd_pq_control);
2940
2941         /* set the wb address whether it's enabled or not */
2942         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
2943                 mqd->cp_hqd_pq_rptr_report_addr_lo);
2944         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
2945                 mqd->cp_hqd_pq_rptr_report_addr_hi);
2946
2947         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2948         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
2949                mqd->cp_hqd_pq_wptr_poll_addr_lo);
2950         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
2951                mqd->cp_hqd_pq_wptr_poll_addr_hi);
2952
2953         /* enable the doorbell if requested */
2954         if (ring->use_doorbell) {
2955                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
2956                         (adev->doorbell_index.kiq * 2) << 2);
2957                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
2958                         (adev->doorbell_index.userqueue_end * 2) << 2);
2959         }
2960
2961         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
2962                mqd->cp_hqd_pq_doorbell_control);
2963
2964         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2965         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
2966                mqd->cp_hqd_pq_wptr_lo);
2967         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
2968                mqd->cp_hqd_pq_wptr_hi);
2969
2970         /* set the vmid for the queue */
2971         WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
2972
2973         WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
2974                mqd->cp_hqd_persistent_state);
2975
2976         /* activate the queue */
2977         WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
2978                mqd->cp_hqd_active);
2979
2980         if (ring->use_doorbell)
2981                 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
2982
2983         return 0;
2984 }
2985
2986 static int gfx_v12_0_kiq_init_queue(struct amdgpu_ring *ring)
2987 {
2988         struct amdgpu_device *adev = ring->adev;
2989         struct v12_compute_mqd *mqd = ring->mqd_ptr;
2990         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
2991
2992         gfx_v12_0_kiq_setting(ring);
2993
2994         if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
2995                 /* reset MQD to a clean status */
2996                 if (adev->gfx.mec.mqd_backup[mqd_idx])
2997                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
2998
2999                 /* reset ring buffer */
3000                 ring->wptr = 0;
3001                 amdgpu_ring_clear_ring(ring);
3002
3003                 mutex_lock(&adev->srbm_mutex);
3004                 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3005                 gfx_v12_0_kiq_init_register(ring);
3006                 soc24_grbm_select(adev, 0, 0, 0, 0);
3007                 mutex_unlock(&adev->srbm_mutex);
3008         } else {
3009                 memset((void *)mqd, 0, sizeof(*mqd));
3010                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3011                         amdgpu_ring_clear_ring(ring);
3012                 mutex_lock(&adev->srbm_mutex);
3013                 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3014                 amdgpu_ring_init_mqd(ring);
3015                 gfx_v12_0_kiq_init_register(ring);
3016                 soc24_grbm_select(adev, 0, 0, 0, 0);
3017                 mutex_unlock(&adev->srbm_mutex);
3018
3019                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3020                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3021         }
3022
3023         return 0;
3024 }
3025
3026 static int gfx_v12_0_kcq_init_queue(struct amdgpu_ring *ring)
3027 {
3028         struct amdgpu_device *adev = ring->adev;
3029         struct v12_compute_mqd *mqd = ring->mqd_ptr;
3030         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3031
3032         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3033                 memset((void *)mqd, 0, sizeof(*mqd));
3034                 mutex_lock(&adev->srbm_mutex);
3035                 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3036                 amdgpu_ring_init_mqd(ring);
3037                 soc24_grbm_select(adev, 0, 0, 0, 0);
3038                 mutex_unlock(&adev->srbm_mutex);
3039
3040                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3041                         memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3042         } else {
3043                 /* restore MQD to a clean status */
3044                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3045                         memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
3046                 /* reset ring buffer */
3047                 ring->wptr = 0;
3048                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
3049                 amdgpu_ring_clear_ring(ring);
3050         }
3051
3052         return 0;
3053 }
3054
3055 static int gfx_v12_0_kiq_resume(struct amdgpu_device *adev)
3056 {
3057         struct amdgpu_ring *ring;
3058         int r;
3059
3060         ring = &adev->gfx.kiq[0].ring;
3061
3062         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3063         if (unlikely(r != 0))
3064                 return r;
3065
3066         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3067         if (unlikely(r != 0)) {
3068                 amdgpu_bo_unreserve(ring->mqd_obj);
3069                 return r;
3070         }
3071
3072         gfx_v12_0_kiq_init_queue(ring);
3073         amdgpu_bo_kunmap(ring->mqd_obj);
3074         ring->mqd_ptr = NULL;
3075         amdgpu_bo_unreserve(ring->mqd_obj);
3076         ring->sched.ready = true;
3077         return 0;
3078 }
3079
3080 static int gfx_v12_0_kcq_resume(struct amdgpu_device *adev)
3081 {
3082         struct amdgpu_ring *ring = NULL;
3083         int r = 0, i;
3084
3085         if (!amdgpu_async_gfx_ring)
3086                 gfx_v12_0_cp_compute_enable(adev, true);
3087
3088         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3089                 ring = &adev->gfx.compute_ring[i];
3090
3091                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3092                 if (unlikely(r != 0))
3093                         goto done;
3094                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3095                 if (!r) {
3096                         r = gfx_v12_0_kcq_init_queue(ring);
3097                         amdgpu_bo_kunmap(ring->mqd_obj);
3098                         ring->mqd_ptr = NULL;
3099                 }
3100                 amdgpu_bo_unreserve(ring->mqd_obj);
3101                 if (r)
3102                         goto done;
3103         }
3104
3105         r = amdgpu_gfx_enable_kcq(adev, 0);
3106 done:
3107         return r;
3108 }
3109
3110 static int gfx_v12_0_cp_resume(struct amdgpu_device *adev)
3111 {
3112         int r, i;
3113         struct amdgpu_ring *ring;
3114
3115         if (!(adev->flags & AMD_IS_APU))
3116                 gfx_v12_0_enable_gui_idle_interrupt(adev, false);
3117
3118         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
3119                 /* legacy firmware loading */
3120                 r = gfx_v12_0_cp_gfx_load_microcode(adev);
3121                 if (r)
3122                         return r;
3123
3124                 r = gfx_v12_0_cp_compute_load_microcode_rs64(adev);
3125                 if (r)
3126                         return r;
3127         }
3128
3129         gfx_v12_0_cp_set_doorbell_range(adev);
3130
3131         if (amdgpu_async_gfx_ring) {
3132                 gfx_v12_0_cp_compute_enable(adev, true);
3133                 gfx_v12_0_cp_gfx_enable(adev, true);
3134         }
3135
3136         if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
3137                 r = amdgpu_mes_kiq_hw_init(adev);
3138         else
3139                 r = gfx_v12_0_kiq_resume(adev);
3140         if (r)
3141                 return r;
3142
3143         r = gfx_v12_0_kcq_resume(adev);
3144         if (r)
3145                 return r;
3146
3147         if (!amdgpu_async_gfx_ring) {
3148                 r = gfx_v12_0_cp_gfx_resume(adev);
3149                 if (r)
3150                         return r;
3151         } else {
3152                 r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
3153                 if (r)
3154                         return r;
3155         }
3156
3157         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3158                 ring = &adev->gfx.gfx_ring[i];
3159                 r = amdgpu_ring_test_helper(ring);
3160                 if (r)
3161                         return r;
3162         }
3163
3164         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3165                 ring = &adev->gfx.compute_ring[i];
3166                 r = amdgpu_ring_test_helper(ring);
3167                 if (r)
3168                         return r;
3169         }
3170
3171         return 0;
3172 }
3173
3174 static void gfx_v12_0_cp_enable(struct amdgpu_device *adev, bool enable)
3175 {
3176         gfx_v12_0_cp_gfx_enable(adev, enable);
3177         gfx_v12_0_cp_compute_enable(adev, enable);
3178 }
3179
3180 static int gfx_v12_0_gfxhub_enable(struct amdgpu_device *adev)
3181 {
3182         int r;
3183         bool value;
3184
3185         r = adev->gfxhub.funcs->gart_enable(adev);
3186         if (r)
3187                 return r;
3188
3189         adev->hdp.funcs->flush_hdp(adev, NULL);
3190
3191         value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
3192                 false : true;
3193
3194         adev->gfxhub.funcs->set_fault_enable_default(adev, value);
3195         /* TODO investigate why this and the hdp flush above is needed,
3196          * are we missing a flush somewhere else? */
3197         adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
3198
3199         return 0;
3200 }
3201
3202 static int get_gb_addr_config(struct amdgpu_device *adev)
3203 {
3204         u32 gb_addr_config;
3205
3206         gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
3207         if (gb_addr_config == 0)
3208                 return -EINVAL;
3209
3210         adev->gfx.config.gb_addr_config_fields.num_pkrs =
3211                 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
3212
3213         adev->gfx.config.gb_addr_config = gb_addr_config;
3214
3215         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
3216                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3217                                       GB_ADDR_CONFIG, NUM_PIPES);
3218
3219         adev->gfx.config.max_tile_pipes =
3220                 adev->gfx.config.gb_addr_config_fields.num_pipes;
3221
3222         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
3223                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3224                                       GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
3225         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
3226                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3227                                       GB_ADDR_CONFIG, NUM_RB_PER_SE);
3228         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
3229                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3230                                       GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
3231         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
3232                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3233                                       GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
3234
3235         return 0;
3236 }
3237
3238 static void gfx_v12_0_disable_gpa_mode(struct amdgpu_device *adev)
3239 {
3240         uint32_t data;
3241
3242         data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
3243         data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
3244         WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
3245
3246         data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
3247         data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
3248         WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
3249 }
3250
3251 static int gfx_v12_0_hw_init(void *handle)
3252 {
3253         int r;
3254         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3255
3256         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
3257                 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
3258                         /* RLC autoload sequence 1: Program rlc ram */
3259                         if (adev->gfx.imu.funcs->program_rlc_ram)
3260                                 adev->gfx.imu.funcs->program_rlc_ram(adev);
3261                 }
3262                 /* rlc autoload firmware */
3263                 r = gfx_v12_0_rlc_backdoor_autoload_enable(adev);
3264                 if (r)
3265                         return r;
3266         } else {
3267                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
3268                         if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
3269                                 if (adev->gfx.imu.funcs->load_microcode)
3270                                         adev->gfx.imu.funcs->load_microcode(adev);
3271                                 if (adev->gfx.imu.funcs->setup_imu)
3272                                         adev->gfx.imu.funcs->setup_imu(adev);
3273                                 if (adev->gfx.imu.funcs->start_imu)
3274                                         adev->gfx.imu.funcs->start_imu(adev);
3275                         }
3276
3277                         /* disable gpa mode in backdoor loading */
3278                         gfx_v12_0_disable_gpa_mode(adev);
3279                 }
3280         }
3281
3282         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
3283             (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
3284                 r = gfx_v12_0_wait_for_rlc_autoload_complete(adev);
3285                 if (r) {
3286                         dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
3287                         return r;
3288                 }
3289         }
3290
3291         adev->gfx.is_poweron = true;
3292
3293         if (get_gb_addr_config(adev))
3294                 DRM_WARN("Invalid gb_addr_config !\n");
3295
3296         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
3297                 gfx_v12_0_config_gfx_rs64(adev);
3298
3299         r = gfx_v12_0_gfxhub_enable(adev);
3300         if (r)
3301                 return r;
3302
3303         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT ||
3304              adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) &&
3305              (amdgpu_dpm == 1)) {
3306                 /**
3307                  * For gfx 12, rlc firmware loading relies on smu firmware is
3308                  * loaded firstly, so in direct type, it has to load smc ucode
3309                  * here before rlc.
3310                  */
3311                 if (!(adev->flags & AMD_IS_APU)) {
3312                         r = amdgpu_pm_load_smu_firmware(adev, NULL);
3313                         if (r)
3314                                 return r;
3315                 }
3316         }
3317
3318         gfx_v12_0_constants_init(adev);
3319
3320         if (adev->nbio.funcs->gc_doorbell_init)
3321                 adev->nbio.funcs->gc_doorbell_init(adev);
3322
3323         r = gfx_v12_0_rlc_resume(adev);
3324         if (r)
3325                 return r;
3326
3327         /*
3328          * init golden registers and rlc resume may override some registers,
3329          * reconfig them here
3330          */
3331         gfx_v12_0_tcp_harvest(adev);
3332
3333         r = gfx_v12_0_cp_resume(adev);
3334         if (r)
3335                 return r;
3336
3337         return r;
3338 }
3339
3340 static int gfx_v12_0_kiq_disable_kgq(struct amdgpu_device *adev)
3341 {
3342         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
3343         struct amdgpu_ring *kiq_ring = &kiq->ring;
3344         int i, r = 0;
3345
3346         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
3347                 return -EINVAL;
3348
3349         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
3350                                         adev->gfx.num_gfx_rings))
3351                 return -ENOMEM;
3352
3353         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
3354                 kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i],
3355                                            PREEMPT_QUEUES, 0, 0);
3356
3357         if (adev->gfx.kiq[0].ring.sched.ready)
3358                 r = amdgpu_ring_test_helper(kiq_ring);
3359
3360         return r;
3361 }
3362
3363 static int gfx_v12_0_hw_fini(void *handle)
3364 {
3365         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3366         int r;
3367         uint32_t tmp;
3368
3369         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3370         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3371
3372         if (!adev->no_hw_access) {
3373                 if (amdgpu_async_gfx_ring) {
3374                         r = gfx_v12_0_kiq_disable_kgq(adev);
3375                         if (r)
3376                                 DRM_ERROR("KGQ disable failed\n");
3377                 }
3378
3379                 if (amdgpu_gfx_disable_kcq(adev, 0))
3380                         DRM_ERROR("KCQ disable failed\n");
3381
3382                 amdgpu_mes_kiq_hw_fini(adev);
3383         }
3384
3385         if (amdgpu_sriov_vf(adev)) {
3386                 gfx_v12_0_cp_gfx_enable(adev, false);
3387                 /* Program KIQ position of RLC_CP_SCHEDULERS during destroy */
3388                 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
3389                 tmp &= 0xffffff00;
3390                 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3391
3392                 return 0;
3393         }
3394         gfx_v12_0_cp_enable(adev, false);
3395         gfx_v12_0_enable_gui_idle_interrupt(adev, false);
3396
3397         adev->gfxhub.funcs->gart_disable(adev);
3398
3399         adev->gfx.is_poweron = false;
3400
3401         return 0;
3402 }
3403
3404 static int gfx_v12_0_suspend(void *handle)
3405 {
3406         return gfx_v12_0_hw_fini(handle);
3407 }
3408
3409 static int gfx_v12_0_resume(void *handle)
3410 {
3411         return gfx_v12_0_hw_init(handle);
3412 }
3413
3414 static bool gfx_v12_0_is_idle(void *handle)
3415 {
3416         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3417
3418         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
3419                                 GRBM_STATUS, GUI_ACTIVE))
3420                 return false;
3421         else
3422                 return true;
3423 }
3424
3425 static int gfx_v12_0_wait_for_idle(void *handle)
3426 {
3427         unsigned i;
3428         u32 tmp;
3429         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3430
3431         for (i = 0; i < adev->usec_timeout; i++) {
3432                 /* read MC_STATUS */
3433                 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
3434                         GRBM_STATUS__GUI_ACTIVE_MASK;
3435
3436                 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
3437                         return 0;
3438                 udelay(1);
3439         }
3440         return -ETIMEDOUT;
3441 }
3442
3443 static uint64_t gfx_v12_0_get_gpu_clock_counter(struct amdgpu_device *adev)
3444 {
3445         uint64_t clock = 0;
3446
3447         if (adev->smuio.funcs &&
3448             adev->smuio.funcs->get_gpu_clock_counter)
3449                 clock = adev->smuio.funcs->get_gpu_clock_counter(adev);
3450         else
3451                 dev_warn(adev->dev, "query gpu clock counter is not supported\n");
3452
3453         return clock;
3454 }
3455
3456 static int gfx_v12_0_early_init(void *handle)
3457 {
3458         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3459
3460         adev->gfx.funcs = &gfx_v12_0_gfx_funcs;
3461
3462         adev->gfx.num_gfx_rings = GFX12_NUM_GFX_RINGS;
3463         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
3464                                           AMDGPU_MAX_COMPUTE_RINGS);
3465
3466         gfx_v12_0_set_kiq_pm4_funcs(adev);
3467         gfx_v12_0_set_ring_funcs(adev);
3468         gfx_v12_0_set_irq_funcs(adev);
3469         gfx_v12_0_set_rlc_funcs(adev);
3470         gfx_v12_0_set_mqd_funcs(adev);
3471         gfx_v12_0_set_imu_funcs(adev);
3472
3473         gfx_v12_0_init_rlcg_reg_access_ctrl(adev);
3474
3475         return gfx_v12_0_init_microcode(adev);
3476 }
3477
3478 static int gfx_v12_0_late_init(void *handle)
3479 {
3480         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3481         int r;
3482
3483         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
3484         if (r)
3485                 return r;
3486
3487         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
3488         if (r)
3489                 return r;
3490
3491         return 0;
3492 }
3493
3494 static bool gfx_v12_0_is_rlc_enabled(struct amdgpu_device *adev)
3495 {
3496         uint32_t rlc_cntl;
3497
3498         /* if RLC is not enabled, do nothing */
3499         rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
3500         return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
3501 }
3502
3503 static void gfx_v12_0_set_safe_mode(struct amdgpu_device *adev,
3504                                     int xcc_id)
3505 {
3506         uint32_t data;
3507         unsigned i;
3508
3509         data = RLC_SAFE_MODE__CMD_MASK;
3510         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
3511
3512         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
3513
3514         /* wait for RLC_SAFE_MODE */
3515         for (i = 0; i < adev->usec_timeout; i++) {
3516                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
3517                                    RLC_SAFE_MODE, CMD))
3518                         break;
3519                 udelay(1);
3520         }
3521 }
3522
3523 static void gfx_v12_0_unset_safe_mode(struct amdgpu_device *adev,
3524                                       int xcc_id)
3525 {
3526         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
3527 }
3528
3529 static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev,
3530                                       bool enable)
3531 {
3532         uint32_t def, data;
3533
3534         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
3535                 return;
3536
3537         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3538
3539         if (enable)
3540                 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
3541         else
3542                 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
3543
3544         if (def != data)
3545                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3546 }
3547
3548 static void gfx_v12_0_update_spm_vmid(struct amdgpu_device *adev,
3549                                       struct amdgpu_ring *ring,
3550                                       unsigned vmid)
3551 {
3552         u32 reg, data;
3553
3554         reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
3555         if (amdgpu_sriov_is_pp_one_vf(adev))
3556                 data = RREG32_NO_KIQ(reg);
3557         else
3558                 data = RREG32(reg);
3559
3560         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
3561         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
3562
3563         if (amdgpu_sriov_is_pp_one_vf(adev))
3564                 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
3565         else
3566                 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
3567
3568         if (ring
3569             && amdgpu_sriov_is_pp_one_vf(adev)
3570             && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
3571                 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
3572                 uint32_t reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
3573                 amdgpu_ring_emit_wreg(ring, reg, data);
3574         }
3575 }
3576
3577 static const struct amdgpu_rlc_funcs gfx_v12_0_rlc_funcs = {
3578         .is_rlc_enabled = gfx_v12_0_is_rlc_enabled,
3579         .set_safe_mode = gfx_v12_0_set_safe_mode,
3580         .unset_safe_mode = gfx_v12_0_unset_safe_mode,
3581         .init = gfx_v12_0_rlc_init,
3582         .get_csb_size = gfx_v12_0_get_csb_size,
3583         .get_csb_buffer = gfx_v12_0_get_csb_buffer,
3584         .resume = gfx_v12_0_rlc_resume,
3585         .stop = gfx_v12_0_rlc_stop,
3586         .reset = gfx_v12_0_rlc_reset,
3587         .start = gfx_v12_0_rlc_start,
3588         .update_spm_vmid = gfx_v12_0_update_spm_vmid,
3589 };
3590
3591 #if 0
3592 static void gfx_v12_cntl_power_gating(struct amdgpu_device *adev, bool enable)
3593 {
3594         /* TODO */
3595 }
3596
3597 static void gfx_v12_cntl_pg(struct amdgpu_device *adev, bool enable)
3598 {
3599         /* TODO */
3600 }
3601 #endif
3602
3603 static int gfx_v12_0_set_powergating_state(void *handle,
3604                                            enum amd_powergating_state state)
3605 {
3606         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3607         bool enable = (state == AMD_PG_STATE_GATE);
3608
3609         if (amdgpu_sriov_vf(adev))
3610                 return 0;
3611
3612         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3613         case IP_VERSION(12, 0, 0):
3614         case IP_VERSION(12, 0, 1):
3615                 amdgpu_gfx_off_ctrl(adev, enable);
3616                 break;
3617         default:
3618                 break;
3619         }
3620
3621         return 0;
3622 }
3623
3624 static void gfx_v12_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
3625                                                        bool enable)
3626 {
3627         uint32_t def, data;
3628
3629         if (!(adev->cg_flags &
3630               (AMD_CG_SUPPORT_GFX_CGCG |
3631               AMD_CG_SUPPORT_GFX_CGLS |
3632               AMD_CG_SUPPORT_GFX_3D_CGCG |
3633               AMD_CG_SUPPORT_GFX_3D_CGLS)))
3634                 return;
3635
3636         if (enable) {
3637                 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3638
3639                 /* unset CGCG override */
3640                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
3641                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
3642                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
3643                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
3644                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
3645                     adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
3646                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
3647
3648                 /* update CGCG override bits */
3649                 if (def != data)
3650                         WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3651
3652                 /* enable cgcg FSM(0x0000363F) */
3653                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
3654
3655                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
3656                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
3657                         data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
3658                                  RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
3659                 }
3660
3661                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
3662                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
3663                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
3664                                  RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
3665                 }
3666
3667                 if (def != data)
3668                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
3669
3670                 /* Program RLC_CGCG_CGLS_CTRL_3D */
3671                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
3672
3673                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
3674                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
3675                         data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
3676                                  RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
3677                 }
3678
3679                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
3680                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
3681                         data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
3682                                  RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
3683                 }
3684
3685                 if (def != data)
3686                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
3687
3688                 /* set IDLE_POLL_COUNT(0x00900100) */
3689                 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
3690
3691                 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
3692                 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
3693                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
3694
3695                 if (def != data)
3696                         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
3697
3698                 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
3699                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
3700                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
3701                 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
3702                 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
3703                 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
3704
3705                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
3706                 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
3707                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
3708
3709                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
3710                 if (adev->sdma.num_instances > 1) {
3711                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
3712                         data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
3713                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
3714                 }
3715         } else {
3716                 /* Program RLC_CGCG_CGLS_CTRL */
3717                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
3718
3719                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
3720                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
3721
3722                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
3723                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
3724
3725                 if (def != data)
3726                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
3727
3728                 /* Program RLC_CGCG_CGLS_CTRL_3D */
3729                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
3730
3731                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
3732                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
3733                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
3734                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
3735
3736                 if (def != data)
3737                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
3738
3739                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
3740                 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
3741                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
3742
3743                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
3744                 if (adev->sdma.num_instances > 1) {
3745                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
3746                         data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
3747                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
3748                 }
3749         }
3750 }
3751
3752 static void gfx_v12_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
3753                                                        bool enable)
3754 {
3755         uint32_t data, def;
3756         if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
3757                 return;
3758
3759         /* It is disabled by HW by default */
3760         if (enable) {
3761                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
3762                         /* 1 - RLC_CGTT_MGCG_OVERRIDE */
3763                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3764
3765                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
3766                                   RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
3767                                   RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
3768
3769                         if (def != data)
3770                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3771                 }
3772         } else {
3773                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
3774                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3775
3776                         data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
3777                                  RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
3778                                  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
3779
3780                         if (def != data)
3781                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3782                 }
3783         }
3784 }
3785
3786 static void gfx_v12_0_update_repeater_fgcg(struct amdgpu_device *adev,
3787                                            bool enable)
3788 {
3789         uint32_t def, data;
3790
3791         if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
3792                 return;
3793
3794         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3795
3796         if (enable)
3797                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK |
3798                                   RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK);
3799         else
3800                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK |
3801                                 RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK;
3802
3803         if (def != data)
3804                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3805 }
3806
3807 static void gfx_v12_0_update_sram_fgcg(struct amdgpu_device *adev,
3808                                        bool enable)
3809 {
3810         uint32_t def, data;
3811
3812         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
3813                 return;
3814
3815         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3816
3817         if (enable)
3818                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
3819         else
3820                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
3821
3822         if (def != data)
3823                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3824 }
3825
3826 static int gfx_v12_0_update_gfx_clock_gating(struct amdgpu_device *adev,
3827                                             bool enable)
3828 {
3829         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
3830
3831         gfx_v12_0_update_coarse_grain_clock_gating(adev, enable);
3832
3833         gfx_v12_0_update_medium_grain_clock_gating(adev, enable);
3834
3835         gfx_v12_0_update_repeater_fgcg(adev, enable);
3836
3837         gfx_v12_0_update_sram_fgcg(adev, enable);
3838
3839         gfx_v12_0_update_perf_clk(adev, enable);
3840
3841         if (adev->cg_flags &
3842             (AMD_CG_SUPPORT_GFX_MGCG |
3843              AMD_CG_SUPPORT_GFX_CGLS |
3844              AMD_CG_SUPPORT_GFX_CGCG |
3845              AMD_CG_SUPPORT_GFX_3D_CGCG |
3846              AMD_CG_SUPPORT_GFX_3D_CGLS))
3847                 gfx_v12_0_enable_gui_idle_interrupt(adev, enable);
3848
3849         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
3850
3851         return 0;
3852 }
3853
3854 static int gfx_v12_0_set_clockgating_state(void *handle,
3855                                            enum amd_clockgating_state state)
3856 {
3857         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3858
3859         if (amdgpu_sriov_vf(adev))
3860                 return 0;
3861
3862         switch (adev->ip_versions[GC_HWIP][0]) {
3863         case IP_VERSION(12, 0, 0):
3864         case IP_VERSION(12, 0, 1):
3865                 gfx_v12_0_update_gfx_clock_gating(adev,
3866                                                   state == AMD_CG_STATE_GATE);
3867                 break;
3868         default:
3869                 break;
3870         }
3871
3872         return 0;
3873 }
3874
3875 static void gfx_v12_0_get_clockgating_state(void *handle, u64 *flags)
3876 {
3877         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3878         int data;
3879
3880         /* AMD_CG_SUPPORT_GFX_MGCG */
3881         data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3882         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
3883                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
3884
3885         /* AMD_CG_SUPPORT_REPEATER_FGCG */
3886         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
3887                 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
3888
3889         /* AMD_CG_SUPPORT_GFX_FGCG */
3890         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
3891                 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
3892
3893         /* AMD_CG_SUPPORT_GFX_PERF_CLK */
3894         if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
3895                 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
3896
3897         /* AMD_CG_SUPPORT_GFX_CGCG */
3898         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
3899         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
3900                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
3901
3902         /* AMD_CG_SUPPORT_GFX_CGLS */
3903         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
3904                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
3905
3906         /* AMD_CG_SUPPORT_GFX_3D_CGCG */
3907         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
3908         if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
3909                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
3910
3911         /* AMD_CG_SUPPORT_GFX_3D_CGLS */
3912         if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
3913                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
3914 }
3915
3916 static u64 gfx_v12_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
3917 {
3918         /* gfx12 is 32bit rptr*/
3919         return *(uint32_t *)ring->rptr_cpu_addr;
3920 }
3921
3922 static u64 gfx_v12_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
3923 {
3924         struct amdgpu_device *adev = ring->adev;
3925         u64 wptr;
3926
3927         /* XXX check if swapping is necessary on BE */
3928         if (ring->use_doorbell) {
3929                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
3930         } else {
3931                 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
3932                 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
3933         }
3934
3935         return wptr;
3936 }
3937
3938 static void gfx_v12_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
3939 {
3940         struct amdgpu_device *adev = ring->adev;
3941         uint32_t *wptr_saved;
3942         uint32_t *is_queue_unmap;
3943         uint64_t aggregated_db_index;
3944         uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_GFX].mqd_size;
3945         uint64_t wptr_tmp;
3946
3947         if (ring->is_mes_queue) {
3948                 wptr_saved = (uint32_t *)(ring->mqd_ptr + mqd_size);
3949                 is_queue_unmap = (uint32_t *)(ring->mqd_ptr + mqd_size +
3950                                               sizeof(uint32_t));
3951                 aggregated_db_index =
3952                         amdgpu_mes_get_aggregated_doorbell_index(adev,
3953                                                                  ring->hw_prio);
3954
3955                 wptr_tmp = ring->wptr & ring->buf_mask;
3956                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, wptr_tmp);
3957                 *wptr_saved = wptr_tmp;
3958                 /* assume doorbell always being used by mes mapped queue */
3959                 if (*is_queue_unmap) {
3960                         WDOORBELL64(aggregated_db_index, wptr_tmp);
3961                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
3962                 } else {
3963                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
3964
3965                         if (*is_queue_unmap)
3966                                 WDOORBELL64(aggregated_db_index, wptr_tmp);
3967                 }
3968         } else {
3969                 if (ring->use_doorbell) {
3970                         /* XXX check if swapping is necessary on BE */
3971                         atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
3972                                      ring->wptr);
3973                         WDOORBELL64(ring->doorbell_index, ring->wptr);
3974                 } else {
3975                         WREG32_SOC15(GC, 0, regCP_RB0_WPTR,
3976                                      lower_32_bits(ring->wptr));
3977                         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI,
3978                                      upper_32_bits(ring->wptr));
3979                 }
3980         }
3981 }
3982
3983 static u64 gfx_v12_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
3984 {
3985         /* gfx12 hardware is 32bit rptr */
3986         return *(uint32_t *)ring->rptr_cpu_addr;
3987 }
3988
3989 static u64 gfx_v12_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
3990 {
3991         u64 wptr;
3992
3993         /* XXX check if swapping is necessary on BE */
3994         if (ring->use_doorbell)
3995                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
3996         else
3997                 BUG();
3998         return wptr;
3999 }
4000
4001 static void gfx_v12_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
4002 {
4003         struct amdgpu_device *adev = ring->adev;
4004         uint32_t *wptr_saved;
4005         uint32_t *is_queue_unmap;
4006         uint64_t aggregated_db_index;
4007         uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size;
4008         uint64_t wptr_tmp;
4009
4010         if (ring->is_mes_queue) {
4011                 wptr_saved = (uint32_t *)(ring->mqd_ptr + mqd_size);
4012                 is_queue_unmap = (uint32_t *)(ring->mqd_ptr + mqd_size +
4013                                               sizeof(uint32_t));
4014                 aggregated_db_index =
4015                         amdgpu_mes_get_aggregated_doorbell_index(adev,
4016                                                                  ring->hw_prio);
4017
4018                 wptr_tmp = ring->wptr & ring->buf_mask;
4019                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, wptr_tmp);
4020                 *wptr_saved = wptr_tmp;
4021                 /* assume doorbell always used by mes mapped queue */
4022                 if (*is_queue_unmap) {
4023                         WDOORBELL64(aggregated_db_index, wptr_tmp);
4024                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
4025                 } else {
4026                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
4027
4028                         if (*is_queue_unmap)
4029                                 WDOORBELL64(aggregated_db_index, wptr_tmp);
4030                 }
4031         } else {
4032                 /* XXX check if swapping is necessary on BE */
4033                 if (ring->use_doorbell) {
4034                         atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
4035                                      ring->wptr);
4036                         WDOORBELL64(ring->doorbell_index, ring->wptr);
4037                 } else {
4038                         BUG(); /* only DOORBELL method supported on gfx12 now */
4039                 }
4040         }
4041 }
4042
4043 static void gfx_v12_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
4044 {
4045         struct amdgpu_device *adev = ring->adev;
4046         u32 ref_and_mask, reg_mem_engine;
4047         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
4048
4049         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
4050                 switch (ring->me) {
4051                 case 1:
4052                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
4053                         break;
4054                 case 2:
4055                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
4056                         break;
4057                 default:
4058                         return;
4059                 }
4060                 reg_mem_engine = 0;
4061         } else {
4062                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
4063                 reg_mem_engine = 1; /* pfp */
4064         }
4065
4066         gfx_v12_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
4067                                adev->nbio.funcs->get_hdp_flush_req_offset(adev),
4068                                adev->nbio.funcs->get_hdp_flush_done_offset(adev),
4069                                ref_and_mask, ref_and_mask, 0x20);
4070 }
4071
4072 static void gfx_v12_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
4073                                        struct amdgpu_job *job,
4074                                        struct amdgpu_ib *ib,
4075                                        uint32_t flags)
4076 {
4077         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
4078         u32 header, control = 0;
4079
4080         BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE);
4081
4082         header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4083
4084         control |= ib->length_dw | (vmid << 24);
4085
4086         if (ring->is_mes_queue)
4087                 /* inherit vmid from mqd */
4088                 control |= 0x400000;
4089
4090         amdgpu_ring_write(ring, header);
4091         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
4092         amdgpu_ring_write(ring,
4093 #ifdef __BIG_ENDIAN
4094                 (2 << 0) |
4095 #endif
4096                 lower_32_bits(ib->gpu_addr));
4097         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
4098         amdgpu_ring_write(ring, control);
4099 }
4100
4101 static void gfx_v12_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
4102                                            struct amdgpu_job *job,
4103                                            struct amdgpu_ib *ib,
4104                                            uint32_t flags)
4105 {
4106         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
4107         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
4108
4109         if (ring->is_mes_queue)
4110                 /* inherit vmid from mqd */
4111                 control |= 0x40000000;
4112
4113         /* Currently, there is a high possibility to get wave ID mismatch
4114          * between ME and GDS, leading to a hw deadlock, because ME generates
4115          * different wave IDs than the GDS expects. This situation happens
4116          * randomly when at least 5 compute pipes use GDS ordered append.
4117          * The wave IDs generated by ME are also wrong after suspend/resume.
4118          * Those are probably bugs somewhere else in the kernel driver.
4119          *
4120          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
4121          * GDS to 0 for this ring (me/pipe).
4122          */
4123         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
4124                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
4125                 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
4126         }
4127
4128         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
4129         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
4130         amdgpu_ring_write(ring,
4131 #ifdef __BIG_ENDIAN
4132                                 (2 << 0) |
4133 #endif
4134                                 lower_32_bits(ib->gpu_addr));
4135         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
4136         amdgpu_ring_write(ring, control);
4137 }
4138
4139 static void gfx_v12_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
4140                                      u64 seq, unsigned flags)
4141 {
4142         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
4143         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
4144
4145         /* RELEASE_MEM - flush caches, send int */
4146         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
4147         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
4148                                  PACKET3_RELEASE_MEM_GCR_GL2_WB |
4149                                  PACKET3_RELEASE_MEM_GCR_GL2_INV |
4150                                  PACKET3_RELEASE_MEM_GCR_GL2_US |
4151                                  PACKET3_RELEASE_MEM_GCR_GL1_INV |
4152                                  PACKET3_RELEASE_MEM_GCR_GLV_INV |
4153                                  PACKET3_RELEASE_MEM_GCR_GLM_INV |
4154                                  PACKET3_RELEASE_MEM_GCR_GLM_WB |
4155                                  PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
4156                                  PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
4157                                  PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
4158         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
4159                                  PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
4160
4161         /*
4162          * the address should be Qword aligned if 64bit write, Dword
4163          * aligned if only send 32bit data low (discard data high)
4164          */
4165         if (write64bit)
4166                 BUG_ON(addr & 0x7);
4167         else
4168                 BUG_ON(addr & 0x3);
4169         amdgpu_ring_write(ring, lower_32_bits(addr));
4170         amdgpu_ring_write(ring, upper_32_bits(addr));
4171         amdgpu_ring_write(ring, lower_32_bits(seq));
4172         amdgpu_ring_write(ring, upper_32_bits(seq));
4173         amdgpu_ring_write(ring, ring->is_mes_queue ?
4174                          (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0);
4175 }
4176
4177 static void gfx_v12_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
4178 {
4179         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
4180         uint32_t seq = ring->fence_drv.sync_seq;
4181         uint64_t addr = ring->fence_drv.gpu_addr;
4182
4183         gfx_v12_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
4184                                upper_32_bits(addr), seq, 0xffffffff, 4);
4185 }
4186
4187 static void gfx_v12_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
4188                                    uint16_t pasid, uint32_t flush_type,
4189                                    bool all_hub, uint8_t dst_sel)
4190 {
4191         amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
4192         amdgpu_ring_write(ring,
4193                           PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
4194                           PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
4195                           PACKET3_INVALIDATE_TLBS_PASID(pasid) |
4196                           PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
4197 }
4198
4199 static void gfx_v12_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
4200                                          unsigned vmid, uint64_t pd_addr)
4201 {
4202         if (ring->is_mes_queue)
4203                 gfx_v12_0_ring_invalidate_tlbs(ring, 0, 0, false, 0);
4204         else
4205                 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
4206
4207         /* compute doesn't have PFP */
4208         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
4209                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4210                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4211                 amdgpu_ring_write(ring, 0x0);
4212         }
4213 }
4214
4215 static void gfx_v12_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
4216                                           u64 seq, unsigned int flags)
4217 {
4218         struct amdgpu_device *adev = ring->adev;
4219
4220         /* we only allocate 32bit for each seq wb address */
4221         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
4222
4223         /* write fence seq to the "addr" */
4224         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4225         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4226                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
4227         amdgpu_ring_write(ring, lower_32_bits(addr));
4228         amdgpu_ring_write(ring, upper_32_bits(addr));
4229         amdgpu_ring_write(ring, lower_32_bits(seq));
4230
4231         if (flags & AMDGPU_FENCE_FLAG_INT) {
4232                 /* set register to trigger INT */
4233                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4234                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4235                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
4236                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
4237                 amdgpu_ring_write(ring, 0);
4238                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
4239         }
4240 }
4241
4242 static void gfx_v12_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
4243                                          uint32_t flags)
4244 {
4245         uint32_t dw2 = 0;
4246
4247         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
4248         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
4249                 /* set load_global_config & load_global_uconfig */
4250                 dw2 |= 0x8001;
4251                 /* set load_cs_sh_regs */
4252                 dw2 |= 0x01000000;
4253                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
4254                 dw2 |= 0x10002;
4255         }
4256
4257         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4258         amdgpu_ring_write(ring, dw2);
4259         amdgpu_ring_write(ring, 0);
4260 }
4261
4262 static unsigned gfx_v12_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
4263                                                    uint64_t addr)
4264 {
4265         unsigned ret;
4266
4267         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
4268         amdgpu_ring_write(ring, lower_32_bits(addr));
4269         amdgpu_ring_write(ring, upper_32_bits(addr));
4270         /* discard following DWs if *cond_exec_gpu_addr==0 */
4271         amdgpu_ring_write(ring, 0);
4272         ret = ring->wptr & ring->buf_mask;
4273         /* patch dummy value later */
4274         amdgpu_ring_write(ring, 0);
4275
4276         return ret;
4277 }
4278
4279 static int gfx_v12_0_ring_preempt_ib(struct amdgpu_ring *ring)
4280 {
4281         int i, r = 0;
4282         struct amdgpu_device *adev = ring->adev;
4283         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
4284         struct amdgpu_ring *kiq_ring = &kiq->ring;
4285         unsigned long flags;
4286
4287         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
4288                 return -EINVAL;
4289
4290         spin_lock_irqsave(&kiq->ring_lock, flags);
4291
4292         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
4293                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4294                 return -ENOMEM;
4295         }
4296
4297         /* assert preemption condition */
4298         amdgpu_ring_set_preempt_cond_exec(ring, false);
4299
4300         /* assert IB preemption, emit the trailing fence */
4301         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
4302                                    ring->trail_fence_gpu_addr,
4303                                    ++ring->trail_seq);
4304         amdgpu_ring_commit(kiq_ring);
4305
4306         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4307
4308         /* poll the trailing fence */
4309         for (i = 0; i < adev->usec_timeout; i++) {
4310                 if (ring->trail_seq ==
4311                     le32_to_cpu(*(ring->trail_fence_cpu_addr)))
4312                         break;
4313                 udelay(1);
4314         }
4315
4316         if (i >= adev->usec_timeout) {
4317                 r = -EINVAL;
4318                 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
4319         }
4320
4321         /* deassert preemption condition */
4322         amdgpu_ring_set_preempt_cond_exec(ring, true);
4323         return r;
4324 }
4325
4326 static void gfx_v12_0_ring_emit_frame_cntl(struct amdgpu_ring *ring,
4327                                            bool start,
4328                                            bool secure)
4329 {
4330         uint32_t v = secure ? FRAME_TMZ : 0;
4331
4332         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
4333         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
4334 }
4335
4336 static void gfx_v12_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
4337                                      uint32_t reg_val_offs)
4338 {
4339         struct amdgpu_device *adev = ring->adev;
4340
4341         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4342         amdgpu_ring_write(ring, 0 |     /* src: register*/
4343                                 (5 << 8) |      /* dst: memory */
4344                                 (1 << 20));     /* write confirm */
4345         amdgpu_ring_write(ring, reg);
4346         amdgpu_ring_write(ring, 0);
4347         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4348                                 reg_val_offs * 4));
4349         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4350                                 reg_val_offs * 4));
4351 }
4352
4353 static void gfx_v12_0_ring_emit_wreg(struct amdgpu_ring *ring,
4354                                      uint32_t reg,
4355                                      uint32_t val)
4356 {
4357         uint32_t cmd = 0;
4358
4359         switch (ring->funcs->type) {
4360         case AMDGPU_RING_TYPE_GFX:
4361                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
4362                 break;
4363         case AMDGPU_RING_TYPE_KIQ:
4364                 cmd = (1 << 16); /* no inc addr */
4365                 break;
4366         default:
4367                 cmd = WR_CONFIRM;
4368                 break;
4369         }
4370         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4371         amdgpu_ring_write(ring, cmd);
4372         amdgpu_ring_write(ring, reg);
4373         amdgpu_ring_write(ring, 0);
4374         amdgpu_ring_write(ring, val);
4375 }
4376
4377 static void gfx_v12_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
4378                                         uint32_t val, uint32_t mask)
4379 {
4380         gfx_v12_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
4381 }
4382
4383 static void gfx_v12_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
4384                                                    uint32_t reg0, uint32_t reg1,
4385                                                    uint32_t ref, uint32_t mask)
4386 {
4387         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
4388
4389         gfx_v12_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
4390                                ref, mask, 0x20);
4391 }
4392
4393 static void gfx_v12_0_ring_soft_recovery(struct amdgpu_ring *ring,
4394                                          unsigned vmid)
4395 {
4396         struct amdgpu_device *adev = ring->adev;
4397         uint32_t value = 0;
4398
4399         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
4400         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
4401         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
4402         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
4403         WREG32_SOC15(GC, 0, regSQ_CMD, value);
4404 }
4405
4406 static void
4407 gfx_v12_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
4408                                       uint32_t me, uint32_t pipe,
4409                                       enum amdgpu_interrupt_state state)
4410 {
4411         uint32_t cp_int_cntl, cp_int_cntl_reg;
4412
4413         if (!me) {
4414                 switch (pipe) {
4415                 case 0:
4416                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
4417                         break;
4418                 default:
4419                         DRM_DEBUG("invalid pipe %d\n", pipe);
4420                         return;
4421                 }
4422         } else {
4423                 DRM_DEBUG("invalid me %d\n", me);
4424                 return;
4425         }
4426
4427         switch (state) {
4428         case AMDGPU_IRQ_STATE_DISABLE:
4429                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
4430                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
4431                                             TIME_STAMP_INT_ENABLE, 0);
4432                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
4433                                             GENERIC0_INT_ENABLE, 0);
4434                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
4435                 break;
4436         case AMDGPU_IRQ_STATE_ENABLE:
4437                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
4438                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
4439                                             TIME_STAMP_INT_ENABLE, 1);
4440                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
4441                                             GENERIC0_INT_ENABLE, 1);
4442                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
4443                 break;
4444         default:
4445                 break;
4446         }
4447 }
4448
4449 static void gfx_v12_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
4450                                                      int me, int pipe,
4451                                                      enum amdgpu_interrupt_state state)
4452 {
4453         u32 mec_int_cntl, mec_int_cntl_reg;
4454
4455         /*
4456          * amdgpu controls only the first MEC. That's why this function only
4457          * handles the setting of interrupts for this specific MEC. All other
4458          * pipes' interrupts are set by amdkfd.
4459          */
4460
4461         if (me == 1) {
4462                 switch (pipe) {
4463                 case 0:
4464                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
4465                         break;
4466                 case 1:
4467                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
4468                         break;
4469                 default:
4470                         DRM_DEBUG("invalid pipe %d\n", pipe);
4471                         return;
4472                 }
4473         } else {
4474                 DRM_DEBUG("invalid me %d\n", me);
4475                 return;
4476         }
4477
4478         switch (state) {
4479         case AMDGPU_IRQ_STATE_DISABLE:
4480                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
4481                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4482                                              TIME_STAMP_INT_ENABLE, 0);
4483                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4484                                              GENERIC0_INT_ENABLE, 0);
4485                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
4486                 break;
4487         case AMDGPU_IRQ_STATE_ENABLE:
4488                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
4489                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4490                                              TIME_STAMP_INT_ENABLE, 1);
4491                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4492                                              GENERIC0_INT_ENABLE, 1);
4493                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
4494                 break;
4495         default:
4496                 break;
4497         }
4498 }
4499
4500 static int gfx_v12_0_set_eop_interrupt_state(struct amdgpu_device *adev,
4501                                             struct amdgpu_irq_src *src,
4502                                             unsigned type,
4503                                             enum amdgpu_interrupt_state state)
4504 {
4505         switch (type) {
4506         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
4507                 gfx_v12_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
4508                 break;
4509         case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
4510                 gfx_v12_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
4511                 break;
4512         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
4513                 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
4514                 break;
4515         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
4516                 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
4517                 break;
4518         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
4519                 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
4520                 break;
4521         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
4522                 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
4523                 break;
4524         default:
4525                 break;
4526         }
4527         return 0;
4528 }
4529
4530 static int gfx_v12_0_eop_irq(struct amdgpu_device *adev,
4531                              struct amdgpu_irq_src *source,
4532                              struct amdgpu_iv_entry *entry)
4533 {
4534         int i;
4535         u8 me_id, pipe_id, queue_id;
4536         struct amdgpu_ring *ring;
4537         uint32_t mes_queue_id = entry->src_data[0];
4538
4539         DRM_DEBUG("IH: CP EOP\n");
4540
4541         if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
4542                 struct amdgpu_mes_queue *queue;
4543
4544                 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
4545
4546                 spin_lock(&adev->mes.queue_id_lock);
4547                 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
4548                 if (queue) {
4549                         DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
4550                         amdgpu_fence_process(queue->ring);
4551                 }
4552                 spin_unlock(&adev->mes.queue_id_lock);
4553         } else {
4554                 me_id = (entry->ring_id & 0x0c) >> 2;
4555                 pipe_id = (entry->ring_id & 0x03) >> 0;
4556                 queue_id = (entry->ring_id & 0x70) >> 4;
4557
4558                 switch (me_id) {
4559                 case 0:
4560                         if (pipe_id == 0)
4561                                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
4562                         else
4563                                 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
4564                         break;
4565                 case 1:
4566                 case 2:
4567                         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4568                                 ring = &adev->gfx.compute_ring[i];
4569                                 /* Per-queue interrupt is supported for MEC starting from VI.
4570                                  * The interrupt can only be enabled/disabled per pipe instead
4571                                  * of per queue.
4572                                  */
4573                                 if ((ring->me == me_id) &&
4574                                     (ring->pipe == pipe_id) &&
4575                                     (ring->queue == queue_id))
4576                                         amdgpu_fence_process(ring);
4577                         }
4578                         break;
4579                 }
4580         }
4581
4582         return 0;
4583 }
4584
4585 static int gfx_v12_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
4586                                               struct amdgpu_irq_src *source,
4587                                               unsigned type,
4588                                               enum amdgpu_interrupt_state state)
4589 {
4590         switch (state) {
4591         case AMDGPU_IRQ_STATE_DISABLE:
4592         case AMDGPU_IRQ_STATE_ENABLE:
4593                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
4594                                       PRIV_REG_INT_ENABLE,
4595                                       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
4596                 break;
4597         default:
4598                 break;
4599         }
4600
4601         return 0;
4602 }
4603
4604 static int gfx_v12_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
4605                                                struct amdgpu_irq_src *source,
4606                                                unsigned type,
4607                                                enum amdgpu_interrupt_state state)
4608 {
4609         switch (state) {
4610         case AMDGPU_IRQ_STATE_DISABLE:
4611         case AMDGPU_IRQ_STATE_ENABLE:
4612                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
4613                                PRIV_INSTR_INT_ENABLE,
4614                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
4615                 break;
4616         default:
4617                 break;
4618         }
4619
4620         return 0;
4621 }
4622
4623 static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev,
4624                                         struct amdgpu_iv_entry *entry)
4625 {
4626         u8 me_id, pipe_id, queue_id;
4627         struct amdgpu_ring *ring;
4628         int i;
4629
4630         me_id = (entry->ring_id & 0x0c) >> 2;
4631         pipe_id = (entry->ring_id & 0x03) >> 0;
4632         queue_id = (entry->ring_id & 0x70) >> 4;
4633
4634         switch (me_id) {
4635         case 0:
4636                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4637                         ring = &adev->gfx.gfx_ring[i];
4638                         /* we only enabled 1 gfx queue per pipe for now */
4639                         if (ring->me == me_id && ring->pipe == pipe_id)
4640                                 drm_sched_fault(&ring->sched);
4641                 }
4642                 break;
4643         case 1:
4644         case 2:
4645                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4646                         ring = &adev->gfx.compute_ring[i];
4647                         if (ring->me == me_id && ring->pipe == pipe_id &&
4648                             ring->queue == queue_id)
4649                                 drm_sched_fault(&ring->sched);
4650                 }
4651                 break;
4652         default:
4653                 BUG();
4654                 break;
4655         }
4656 }
4657
4658 static int gfx_v12_0_priv_reg_irq(struct amdgpu_device *adev,
4659                                   struct amdgpu_irq_src *source,
4660                                   struct amdgpu_iv_entry *entry)
4661 {
4662         DRM_ERROR("Illegal register access in command stream\n");
4663         gfx_v12_0_handle_priv_fault(adev, entry);
4664         return 0;
4665 }
4666
4667 static int gfx_v12_0_priv_inst_irq(struct amdgpu_device *adev,
4668                                    struct amdgpu_irq_src *source,
4669                                    struct amdgpu_iv_entry *entry)
4670 {
4671         DRM_ERROR("Illegal instruction in command stream\n");
4672         gfx_v12_0_handle_priv_fault(adev, entry);
4673         return 0;
4674 }
4675
4676 static void gfx_v12_0_emit_mem_sync(struct amdgpu_ring *ring)
4677 {
4678         const unsigned int gcr_cntl =
4679                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
4680                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
4681                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
4682                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
4683                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
4684                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
4685                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
4686                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
4687
4688         /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
4689         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
4690         amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
4691         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
4692         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
4693         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
4694         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
4695         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
4696         amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
4697 }
4698
4699 static const struct amd_ip_funcs gfx_v12_0_ip_funcs = {
4700         .name = "gfx_v12_0",
4701         .early_init = gfx_v12_0_early_init,
4702         .late_init = gfx_v12_0_late_init,
4703         .sw_init = gfx_v12_0_sw_init,
4704         .sw_fini = gfx_v12_0_sw_fini,
4705         .hw_init = gfx_v12_0_hw_init,
4706         .hw_fini = gfx_v12_0_hw_fini,
4707         .suspend = gfx_v12_0_suspend,
4708         .resume = gfx_v12_0_resume,
4709         .is_idle = gfx_v12_0_is_idle,
4710         .wait_for_idle = gfx_v12_0_wait_for_idle,
4711         .set_clockgating_state = gfx_v12_0_set_clockgating_state,
4712         .set_powergating_state = gfx_v12_0_set_powergating_state,
4713         .get_clockgating_state = gfx_v12_0_get_clockgating_state,
4714 };
4715
4716 static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_gfx = {
4717         .type = AMDGPU_RING_TYPE_GFX,
4718         .align_mask = 0xff,
4719         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4720         .support_64bit_ptrs = true,
4721         .secure_submission_supported = true,
4722         .get_rptr = gfx_v12_0_ring_get_rptr_gfx,
4723         .get_wptr = gfx_v12_0_ring_get_wptr_gfx,
4724         .set_wptr = gfx_v12_0_ring_set_wptr_gfx,
4725         .emit_frame_size = /* totally 242 maximum if 16 IBs */
4726                 5 + /* COND_EXEC */
4727                 7 + /* PIPELINE_SYNC */
4728                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4729                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4730                 2 + /* VM_FLUSH */
4731                 8 + /* FENCE for VM_FLUSH */
4732                 20 + /* GDS switch */
4733                 5 + /* COND_EXEC */
4734                 7 + /* HDP_flush */
4735                 4 + /* VGT_flush */
4736                 31 + /* DE_META */
4737                 3 + /* CNTX_CTRL */
4738                 5 + /* HDP_INVL */
4739                 8 + 8 + /* FENCE x2 */
4740                 8, /* gfx_v12_0_emit_mem_sync */
4741         .emit_ib_size = 4, /* gfx_v12_0_ring_emit_ib_gfx */
4742         .emit_ib = gfx_v12_0_ring_emit_ib_gfx,
4743         .emit_fence = gfx_v12_0_ring_emit_fence,
4744         .emit_pipeline_sync = gfx_v12_0_ring_emit_pipeline_sync,
4745         .emit_vm_flush = gfx_v12_0_ring_emit_vm_flush,
4746         .emit_hdp_flush = gfx_v12_0_ring_emit_hdp_flush,
4747         .test_ring = gfx_v12_0_ring_test_ring,
4748         .test_ib = gfx_v12_0_ring_test_ib,
4749         .insert_nop = amdgpu_ring_insert_nop,
4750         .pad_ib = amdgpu_ring_generic_pad_ib,
4751         .emit_cntxcntl = gfx_v12_0_ring_emit_cntxcntl,
4752         .init_cond_exec = gfx_v12_0_ring_emit_init_cond_exec,
4753         .preempt_ib = gfx_v12_0_ring_preempt_ib,
4754         .emit_frame_cntl = gfx_v12_0_ring_emit_frame_cntl,
4755         .emit_wreg = gfx_v12_0_ring_emit_wreg,
4756         .emit_reg_wait = gfx_v12_0_ring_emit_reg_wait,
4757         .emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait,
4758         .soft_recovery = gfx_v12_0_ring_soft_recovery,
4759         .emit_mem_sync = gfx_v12_0_emit_mem_sync,
4760 };
4761
4762 static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_compute = {
4763         .type = AMDGPU_RING_TYPE_COMPUTE,
4764         .align_mask = 0xff,
4765         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4766         .support_64bit_ptrs = true,
4767         .get_rptr = gfx_v12_0_ring_get_rptr_compute,
4768         .get_wptr = gfx_v12_0_ring_get_wptr_compute,
4769         .set_wptr = gfx_v12_0_ring_set_wptr_compute,
4770         .emit_frame_size =
4771                 7 + /* gfx_v12_0_ring_emit_hdp_flush */
4772                 5 + /* hdp invalidate */
4773                 7 + /* gfx_v12_0_ring_emit_pipeline_sync */
4774                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4775                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4776                 2 + /* gfx_v12_0_ring_emit_vm_flush */
4777                 8 + 8 + 8 + /* gfx_v12_0_ring_emit_fence x3 for user fence, vm fence */
4778                 8, /* gfx_v12_0_emit_mem_sync */
4779         .emit_ib_size = 7, /* gfx_v12_0_ring_emit_ib_compute */
4780         .emit_ib = gfx_v12_0_ring_emit_ib_compute,
4781         .emit_fence = gfx_v12_0_ring_emit_fence,
4782         .emit_pipeline_sync = gfx_v12_0_ring_emit_pipeline_sync,
4783         .emit_vm_flush = gfx_v12_0_ring_emit_vm_flush,
4784         .emit_hdp_flush = gfx_v12_0_ring_emit_hdp_flush,
4785         .test_ring = gfx_v12_0_ring_test_ring,
4786         .test_ib = gfx_v12_0_ring_test_ib,
4787         .insert_nop = amdgpu_ring_insert_nop,
4788         .pad_ib = amdgpu_ring_generic_pad_ib,
4789         .emit_wreg = gfx_v12_0_ring_emit_wreg,
4790         .emit_reg_wait = gfx_v12_0_ring_emit_reg_wait,
4791         .emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait,
4792         .emit_mem_sync = gfx_v12_0_emit_mem_sync,
4793 };
4794
4795 static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_kiq = {
4796         .type = AMDGPU_RING_TYPE_KIQ,
4797         .align_mask = 0xff,
4798         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4799         .support_64bit_ptrs = true,
4800         .get_rptr = gfx_v12_0_ring_get_rptr_compute,
4801         .get_wptr = gfx_v12_0_ring_get_wptr_compute,
4802         .set_wptr = gfx_v12_0_ring_set_wptr_compute,
4803         .emit_frame_size =
4804                 7 + /* gfx_v12_0_ring_emit_hdp_flush */
4805                 5 + /*hdp invalidate */
4806                 7 + /* gfx_v12_0_ring_emit_pipeline_sync */
4807                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4808                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4809                 2 + /* gfx_v12_0_ring_emit_vm_flush */
4810                 8 + 8 + 8, /* gfx_v12_0_ring_emit_fence_kiq x3 for user fence, vm fence */
4811         .emit_ib_size = 7, /* gfx_v12_0_ring_emit_ib_compute */
4812         .emit_ib = gfx_v12_0_ring_emit_ib_compute,
4813         .emit_fence = gfx_v12_0_ring_emit_fence_kiq,
4814         .test_ring = gfx_v12_0_ring_test_ring,
4815         .test_ib = gfx_v12_0_ring_test_ib,
4816         .insert_nop = amdgpu_ring_insert_nop,
4817         .pad_ib = amdgpu_ring_generic_pad_ib,
4818         .emit_rreg = gfx_v12_0_ring_emit_rreg,
4819         .emit_wreg = gfx_v12_0_ring_emit_wreg,
4820         .emit_reg_wait = gfx_v12_0_ring_emit_reg_wait,
4821         .emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait,
4822 };
4823
4824 static void gfx_v12_0_set_ring_funcs(struct amdgpu_device *adev)
4825 {
4826         int i;
4827
4828         adev->gfx.kiq[0].ring.funcs = &gfx_v12_0_ring_funcs_kiq;
4829
4830         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
4831                 adev->gfx.gfx_ring[i].funcs = &gfx_v12_0_ring_funcs_gfx;
4832
4833         for (i = 0; i < adev->gfx.num_compute_rings; i++)
4834                 adev->gfx.compute_ring[i].funcs = &gfx_v12_0_ring_funcs_compute;
4835 }
4836
4837 static const struct amdgpu_irq_src_funcs gfx_v12_0_eop_irq_funcs = {
4838         .set = gfx_v12_0_set_eop_interrupt_state,
4839         .process = gfx_v12_0_eop_irq,
4840 };
4841
4842 static const struct amdgpu_irq_src_funcs gfx_v12_0_priv_reg_irq_funcs = {
4843         .set = gfx_v12_0_set_priv_reg_fault_state,
4844         .process = gfx_v12_0_priv_reg_irq,
4845 };
4846
4847 static const struct amdgpu_irq_src_funcs gfx_v12_0_priv_inst_irq_funcs = {
4848         .set = gfx_v12_0_set_priv_inst_fault_state,
4849         .process = gfx_v12_0_priv_inst_irq,
4850 };
4851
4852 static void gfx_v12_0_set_irq_funcs(struct amdgpu_device *adev)
4853 {
4854         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
4855         adev->gfx.eop_irq.funcs = &gfx_v12_0_eop_irq_funcs;
4856
4857         adev->gfx.priv_reg_irq.num_types = 1;
4858         adev->gfx.priv_reg_irq.funcs = &gfx_v12_0_priv_reg_irq_funcs;
4859
4860         adev->gfx.priv_inst_irq.num_types = 1;
4861         adev->gfx.priv_inst_irq.funcs = &gfx_v12_0_priv_inst_irq_funcs;
4862 }
4863
4864 static void gfx_v12_0_set_imu_funcs(struct amdgpu_device *adev)
4865 {
4866         if (adev->flags & AMD_IS_APU)
4867                 adev->gfx.imu.mode = MISSION_MODE;
4868         else
4869                 adev->gfx.imu.mode = DEBUG_MODE;
4870
4871         adev->gfx.imu.funcs = &gfx_v12_0_imu_funcs;
4872 }
4873
4874 static void gfx_v12_0_set_rlc_funcs(struct amdgpu_device *adev)
4875 {
4876         adev->gfx.rlc.funcs = &gfx_v12_0_rlc_funcs;
4877 }
4878
4879 static void gfx_v12_0_set_mqd_funcs(struct amdgpu_device *adev)
4880 {
4881         /* set gfx eng mqd */
4882         adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
4883                 sizeof(struct v12_gfx_mqd);
4884         adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
4885                 gfx_v12_0_gfx_mqd_init;
4886         /* set compute eng mqd */
4887         adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
4888                 sizeof(struct v12_compute_mqd);
4889         adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
4890                 gfx_v12_0_compute_mqd_init;
4891 }
4892
4893 static void gfx_v12_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
4894                                                           u32 bitmap)
4895 {
4896         u32 data;
4897
4898         if (!bitmap)
4899                 return;
4900
4901         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
4902         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
4903
4904         WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
4905 }
4906
4907 static u32 gfx_v12_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
4908 {
4909         u32 data, wgp_bitmask;
4910         data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
4911         data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
4912
4913         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
4914         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
4915
4916         wgp_bitmask =
4917                 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
4918
4919         return (~data) & wgp_bitmask;
4920 }
4921
4922 static u32 gfx_v12_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
4923 {
4924         u32 wgp_idx, wgp_active_bitmap;
4925         u32 cu_bitmap_per_wgp, cu_active_bitmap;
4926
4927         wgp_active_bitmap = gfx_v12_0_get_wgp_active_bitmap_per_sh(adev);
4928         cu_active_bitmap = 0;
4929
4930         for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
4931                 /* if there is one WGP enabled, it means 2 CUs will be enabled */
4932                 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
4933                 if (wgp_active_bitmap & (1 << wgp_idx))
4934                         cu_active_bitmap |= cu_bitmap_per_wgp;
4935         }
4936
4937         return cu_active_bitmap;
4938 }
4939
4940 static int gfx_v12_0_get_cu_info(struct amdgpu_device *adev,
4941                                  struct amdgpu_cu_info *cu_info)
4942 {
4943         int i, j, k, counter, active_cu_number = 0;
4944         u32 mask, bitmap;
4945         unsigned disable_masks[8 * 2];
4946
4947         if (!adev || !cu_info)
4948                 return -EINVAL;
4949
4950         amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
4951
4952         mutex_lock(&adev->grbm_idx_mutex);
4953         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
4954                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
4955                         bitmap = i * adev->gfx.config.max_sh_per_se + j;
4956                         if (!((gfx_v12_0_get_sa_active_bitmap(adev) >> bitmap) & 1))
4957                                 continue;
4958                         mask = 1;
4959                         counter = 0;
4960                         gfx_v12_0_select_se_sh(adev, i, j, 0xffffffff, 0);
4961                         if (i < 8 && j < 2)
4962                                 gfx_v12_0_set_user_wgp_inactive_bitmap_per_sh(
4963                                         adev, disable_masks[i * 2 + j]);
4964                         bitmap = gfx_v12_0_get_cu_active_bitmap_per_sh(adev);
4965
4966                         /**
4967                          * GFX12 could support more than 4 SEs, while the bitmap
4968                          * in cu_info struct is 4x4 and ioctl interface struct
4969                          * drm_amdgpu_info_device should keep stable.
4970                          * So we use last two columns of bitmap to store cu mask for
4971                          * SEs 4 to 7, the layout of the bitmap is as below:
4972                          *    SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
4973                          *    SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
4974                          *    SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
4975                          *    SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
4976                          *    SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
4977                          *    SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
4978                          *    SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
4979                          *    SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
4980                          */
4981                         cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap;
4982
4983                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
4984                                 if (bitmap & mask)
4985                                         counter++;
4986
4987                                 mask <<= 1;
4988                         }
4989                         active_cu_number += counter;
4990                 }
4991         }
4992         gfx_v12_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
4993         mutex_unlock(&adev->grbm_idx_mutex);
4994
4995         cu_info->number = active_cu_number;
4996         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
4997
4998         return 0;
4999 }
5000
5001 const struct amdgpu_ip_block_version gfx_v12_0_ip_block = {
5002         .type = AMD_IP_BLOCK_TYPE_GFX,
5003         .major = 12,
5004         .minor = 0,
5005         .rev = 0,
5006         .funcs = &gfx_v12_0_ip_funcs,
5007 };