drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c

   1 /*
   2  * Copyright 2023 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <linux/delay.h>
  24 #include <linux/kernel.h>
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_gfx.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_smu.h"
  32 #include "amdgpu_atomfirmware.h"
  33 #include "imu_v12_0.h"
  34 #include "soc24.h"
  35 #include "nvd.h"
  36
  37 #include "gc/gc_12_0_0_offset.h"
  38 #include "gc/gc_12_0_0_sh_mask.h"
  39 #include "soc24_enum.h"
  40 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
  41
  42 #include "soc15.h"
  43 #include "soc15d.h"
  44 #include "clearstate_gfx12.h"
  45 #include "v12_structs.h"
  46 #include "gfx_v12_0.h"
  47 #include "nbif_v6_3_1.h"
  48 #include "mes_v12_0.h"
  49
  50 #define GFX12_NUM_GFX_RINGS     1
  51 #define GFX12_MEC_HPD_SIZE      2048
  52
  53 #define RLCG_UCODE_LOADING_START_ADDRESS        0x00002000L
  54
  55 MODULE_FIRMWARE("amdgpu/gc_12_0_0_pfp.bin");
  56 MODULE_FIRMWARE("amdgpu/gc_12_0_0_me.bin");
  57 MODULE_FIRMWARE("amdgpu/gc_12_0_0_mec.bin");
  58 MODULE_FIRMWARE("amdgpu/gc_12_0_0_rlc.bin");
  59 MODULE_FIRMWARE("amdgpu/gc_12_0_0_toc.bin");
  60 MODULE_FIRMWARE("amdgpu/gc_12_0_1_pfp.bin");
  61 MODULE_FIRMWARE("amdgpu/gc_12_0_1_me.bin");
  62 MODULE_FIRMWARE("amdgpu/gc_12_0_1_mec.bin");
  63 MODULE_FIRMWARE("amdgpu/gc_12_0_1_rlc.bin");
  64 MODULE_FIRMWARE("amdgpu/gc_12_0_1_toc.bin");
  65
  66 static const struct amdgpu_hwip_reg_entry gc_reg_list_12_0[] = {
  67         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
  68         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
  69         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3),
  70         SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
  71         SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
  72         SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3),
  73         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
  74         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
  75         SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
  76         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
  77         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
  78         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2),
  79         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2),
  80         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
  81         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
  82         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0),
  83         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE),
  84         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
  85         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR),
  86         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE),
  87         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR),
  88         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR),
  89         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
  90         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ),
  91         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
  92         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
  93         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
  94         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO),
  95         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI),
  96         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ),
  97         SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
  98         SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
  99         SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
 100         SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS),
 101         SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2),
 102         SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS),
 103         SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
 104         SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES),
 105         SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS),
 106         SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
 107         SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL),
 108         SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS_LO32),
 109         SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS_HI32),
 110         SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
 111         SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
 112         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL),
 113         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR),
 114         SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR),
 115         SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR),
 116         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
 117         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_RS64_INSTR_PNTR0),
 118         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_RS64_INSTR_PNTR1),
 119         SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_RS64_INSTR_PNTR),
 120
 121         /* cp header registers */
 122         SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
 123         SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
 124         SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
 125         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
 126         /* SE status registers */
 127         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
 128         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
 129         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2),
 130         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3)
 131 };
 132
 133 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_12[] = {
 134         /* compute registers */
 135         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
 136         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
 137         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
 138         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
 139         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
 140         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
 141         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
 142         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
 143         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
 144         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
 145         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
 146         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
 147         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
 148         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
 149         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
 150         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
 151         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
 152         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
 153         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
 154         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
 155         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
 156         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
 157         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
 158         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
 159         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
 160         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
 161         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
 162         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
 163         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
 164         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
 165         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
 166         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
 167         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
 168         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
 169         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
 170         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET),
 171         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT),
 172         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET),
 173         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS)
 174 };
 175
 176 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_12[] = {
 177         /* gfx queue registers */
 178         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE),
 179         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID),
 180         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY),
 181         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM),
 182         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE),
 183         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI),
 184         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET),
 185         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL),
 186         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR),
 187         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR),
 188         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI),
 189         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST),
 190         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED),
 191         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL),
 192         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0),
 193         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0),
 194         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR),
 195         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI),
 196         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO),
 197         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI),
 198         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
 199         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
 200         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
 201         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
 202         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ)
 203 };
 204
 205 #define DEFAULT_SH_MEM_CONFIG \
 206         ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
 207          (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
 208          (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
 209
 210 static void gfx_v12_0_disable_gpa_mode(struct amdgpu_device *adev);
 211 static void gfx_v12_0_set_ring_funcs(struct amdgpu_device *adev);
 212 static void gfx_v12_0_set_irq_funcs(struct amdgpu_device *adev);
 213 static void gfx_v12_0_set_rlc_funcs(struct amdgpu_device *adev);
 214 static void gfx_v12_0_set_mqd_funcs(struct amdgpu_device *adev);
 215 static void gfx_v12_0_set_imu_funcs(struct amdgpu_device *adev);
 216 static int gfx_v12_0_get_cu_info(struct amdgpu_device *adev,
 217                                  struct amdgpu_cu_info *cu_info);
 218 static uint64_t gfx_v12_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 219 static void gfx_v12_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
 220                                    u32 sh_num, u32 instance, int xcc_id);
 221 static u32 gfx_v12_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
 222
 223 static void gfx_v12_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
 224 static void gfx_v12_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
 225                                      uint32_t val);
 226 static int gfx_v12_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
 227 static void gfx_v12_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
 228                                            uint16_t pasid, uint32_t flush_type,
 229                                            bool all_hub, uint8_t dst_sel);
 230 static void gfx_v12_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
 231 static void gfx_v12_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
 232 static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev,
 233                                       bool enable);
 234
 235 static void gfx_v12_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 236                                         uint64_t queue_mask)
 237 {
 238         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 239         amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
 240                           PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
 241         amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
 242         amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
 243         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 244         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 245         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 246         amdgpu_ring_write(kiq_ring, 0);
 247 }
 248
 249 static void gfx_v12_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 250                                      struct amdgpu_ring *ring)
 251 {
 252         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 253         uint64_t wptr_addr = ring->wptr_gpu_addr;
 254         uint32_t me = 0, eng_sel = 0;
 255
 256         switch (ring->funcs->type) {
 257         case AMDGPU_RING_TYPE_COMPUTE:
 258                 me = 1;
 259                 eng_sel = 0;
 260                 break;
 261         case AMDGPU_RING_TYPE_GFX:
 262                 me = 0;
 263                 eng_sel = 4;
 264                 break;
 265         case AMDGPU_RING_TYPE_MES:
 266                 me = 2;
 267                 eng_sel = 5;
 268                 break;
 269         default:
 270                 WARN_ON(1);
 271         }
 272
 273         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 274         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 275         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 276                           PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 277                           PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 278                           PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 279                           PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 280                           PACKET3_MAP_QUEUES_ME((me)) |
 281                           PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
 282                           PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
 283                           PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 284                           PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
 285         amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 286         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 287         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 288         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 289         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 290 }
 291
 292 static void gfx_v12_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 293                                        struct amdgpu_ring *ring,
 294                                        enum amdgpu_unmap_queues_action action,
 295                                        u64 gpu_addr, u64 seq)
 296 {
 297         struct amdgpu_device *adev = kiq_ring->adev;
 298         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 299
 300         if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
 301                 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
 302                 return;
 303         }
 304
 305         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 306         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 307                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 308                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 309                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 310                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 311         amdgpu_ring_write(kiq_ring,
 312                   PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 313
 314         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 315                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 316                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 317                 amdgpu_ring_write(kiq_ring, seq);
 318         } else {
 319                 amdgpu_ring_write(kiq_ring, 0);
 320                 amdgpu_ring_write(kiq_ring, 0);
 321                 amdgpu_ring_write(kiq_ring, 0);
 322         }
 323 }
 324
 325 static void gfx_v12_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 326                                        struct amdgpu_ring *ring,
 327                                        u64 addr, u64 seq)
 328 {
 329         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 330
 331         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 332         amdgpu_ring_write(kiq_ring,
 333                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 334                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 335                           PACKET3_QUERY_STATUS_COMMAND(2));
 336         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 337                           PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 338                           PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 339         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 340         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 341         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 342         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 343 }
 344
 345 static void gfx_v12_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 346                                           uint16_t pasid,
 347                                           uint32_t flush_type,
 348                                           bool all_hub)
 349 {
 350         gfx_v12_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
 351 }
 352
 353 static const struct kiq_pm4_funcs gfx_v12_0_kiq_pm4_funcs = {
 354         .kiq_set_resources = gfx_v12_0_kiq_set_resources,
 355         .kiq_map_queues = gfx_v12_0_kiq_map_queues,
 356         .kiq_unmap_queues = gfx_v12_0_kiq_unmap_queues,
 357         .kiq_query_status = gfx_v12_0_kiq_query_status,
 358         .kiq_invalidate_tlbs = gfx_v12_0_kiq_invalidate_tlbs,
 359         .set_resources_size = 8,
 360         .map_queues_size = 7,
 361         .unmap_queues_size = 6,
 362         .query_status_size = 7,
 363         .invalidate_tlbs_size = 2,
 364 };
 365
 366 static void gfx_v12_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 367 {
 368         adev->gfx.kiq[0].pmf = &gfx_v12_0_kiq_pm4_funcs;
 369 }
 370
 371 static void gfx_v12_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 372                                    int mem_space, int opt, uint32_t addr0,
 373                                    uint32_t addr1, uint32_t ref,
 374                                    uint32_t mask, uint32_t inv)
 375 {
 376         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 377         amdgpu_ring_write(ring,
 378                           /* memory (1) or register (0) */
 379                           (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 380                            WAIT_REG_MEM_OPERATION(opt) | /* wait */
 381                            WAIT_REG_MEM_FUNCTION(3) |  /* equal */
 382                            WAIT_REG_MEM_ENGINE(eng_sel)));
 383
 384         if (mem_space)
 385                 BUG_ON(addr0 & 0x3); /* Dword align */
 386         amdgpu_ring_write(ring, addr0);
 387         amdgpu_ring_write(ring, addr1);
 388         amdgpu_ring_write(ring, ref);
 389         amdgpu_ring_write(ring, mask);
 390         amdgpu_ring_write(ring, inv); /* poll interval */
 391 }
 392
 393 static int gfx_v12_0_ring_test_ring(struct amdgpu_ring *ring)
 394 {
 395         struct amdgpu_device *adev = ring->adev;
 396         uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 397         uint32_t tmp = 0;
 398         unsigned i;
 399         int r;
 400
 401         WREG32(scratch, 0xCAFEDEAD);
 402         r = amdgpu_ring_alloc(ring, 5);
 403         if (r) {
 404                 dev_err(adev->dev,
 405                         "amdgpu: cp failed to lock ring %d (%d).\n",
 406                         ring->idx, r);
 407                 return r;
 408         }
 409
 410         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
 411                 gfx_v12_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
 412         } else {
 413                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 414                 amdgpu_ring_write(ring, scratch -
 415                                   PACKET3_SET_UCONFIG_REG_START);
 416                 amdgpu_ring_write(ring, 0xDEADBEEF);
 417         }
 418         amdgpu_ring_commit(ring);
 419
 420         for (i = 0; i < adev->usec_timeout; i++) {
 421                 tmp = RREG32(scratch);
 422                 if (tmp == 0xDEADBEEF)
 423                         break;
 424                 if (amdgpu_emu_mode == 1)
 425                         msleep(1);
 426                 else
 427                         udelay(1);
 428         }
 429
 430         if (i >= adev->usec_timeout)
 431                 r = -ETIMEDOUT;
 432         return r;
 433 }
 434
 435 static int gfx_v12_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 436 {
 437         struct amdgpu_device *adev = ring->adev;
 438         struct amdgpu_ib ib;
 439         struct dma_fence *f = NULL;
 440         unsigned index;
 441         uint64_t gpu_addr;
 442         volatile uint32_t *cpu_ptr;
 443         long r;
 444
 445         /* MES KIQ fw hasn't indirect buffer support for now */
 446         if (adev->enable_mes_kiq &&
 447             ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 448                 return 0;
 449
 450         memset(&ib, 0, sizeof(ib));
 451
 452         if (ring->is_mes_queue) {
 453                 uint32_t padding, offset;
 454
 455                 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
 456                 padding = amdgpu_mes_ctx_get_offs(ring,
 457                                                   AMDGPU_MES_CTX_PADDING_OFFS);
 458
 459                 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 460                 ib.ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 461
 462                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, padding);
 463                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, padding);
 464                 *cpu_ptr = cpu_to_le32(0xCAFEDEAD);
 465         } else {
 466                 r = amdgpu_device_wb_get(adev, &index);
 467                 if (r)
 468                         return r;
 469
 470                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 471                 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 472                 cpu_ptr = &adev->wb.wb[index];
 473
 474                 r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
 475                 if (r) {
 476                         dev_err(adev->dev, "amdgpu: failed to get ib (%ld).\n", r);
 477                         goto err1;
 478                 }
 479         }
 480
 481         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
 482         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
 483         ib.ptr[2] = lower_32_bits(gpu_addr);
 484         ib.ptr[3] = upper_32_bits(gpu_addr);
 485         ib.ptr[4] = 0xDEADBEEF;
 486         ib.length_dw = 5;
 487
 488         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 489         if (r)
 490                 goto err2;
 491
 492         r = dma_fence_wait_timeout(f, false, timeout);
 493         if (r == 0) {
 494                 r = -ETIMEDOUT;
 495                 goto err2;
 496         } else if (r < 0) {
 497                 goto err2;
 498         }
 499
 500         if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
 501                 r = 0;
 502         else
 503                 r = -EINVAL;
 504 err2:
 505         if (!ring->is_mes_queue)
 506                 amdgpu_ib_free(adev, &ib, NULL);
 507         dma_fence_put(f);
 508 err1:
 509         if (!ring->is_mes_queue)
 510                 amdgpu_device_wb_free(adev, index);
 511         return r;
 512 }
 513
 514 static void gfx_v12_0_free_microcode(struct amdgpu_device *adev)
 515 {
 516         amdgpu_ucode_release(&adev->gfx.pfp_fw);
 517         amdgpu_ucode_release(&adev->gfx.me_fw);
 518         amdgpu_ucode_release(&adev->gfx.rlc_fw);
 519         amdgpu_ucode_release(&adev->gfx.mec_fw);
 520
 521         kfree(adev->gfx.rlc.register_list_format);
 522 }
 523
 524 static int gfx_v12_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
 525 {
 526         const struct psp_firmware_header_v1_0 *toc_hdr;
 527         int err = 0;
 528
 529         err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
 530                                    "amdgpu/%s_toc.bin", ucode_prefix);
 531         if (err)
 532                 goto out;
 533
 534         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
 535         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
 536         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
 537         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
 538         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
 539                         le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
 540         return 0;
 541 out:
 542         amdgpu_ucode_release(&adev->psp.toc_fw);
 543         return err;
 544 }
 545
 546 static int gfx_v12_0_init_microcode(struct amdgpu_device *adev)
 547 {
 548         char ucode_prefix[15];
 549         int err;
 550         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 551         uint16_t version_major;
 552         uint16_t version_minor;
 553
 554         DRM_DEBUG("\n");
 555
 556         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
 557
 558         err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
 559                                    "amdgpu/%s_pfp.bin", ucode_prefix);
 560         if (err)
 561                 goto out;
 562         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
 563         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
 564
 565         err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
 566                                    "amdgpu/%s_me.bin", ucode_prefix);
 567         if (err)
 568                 goto out;
 569         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
 570         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
 571
 572         if (!amdgpu_sriov_vf(adev)) {
 573                 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
 574                                            "amdgpu/%s_rlc.bin", ucode_prefix);
 575                 if (err)
 576                         goto out;
 577                 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 578                 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 579                 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
 580                 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
 581                 if (err)
 582                         goto out;
 583         }
 584
 585         err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
 586                                    "amdgpu/%s_mec.bin", ucode_prefix);
 587         if (err)
 588                 goto out;
 589         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
 590         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
 591         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
 592
 593         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
 594                 err = gfx_v12_0_init_toc_microcode(adev, ucode_prefix);
 595
 596         /* only one MEC for gfx 12 */
 597         adev->gfx.mec2_fw = NULL;
 598
 599         if (adev->gfx.imu.funcs) {
 600                 if (adev->gfx.imu.funcs->init_microcode) {
 601                         err = adev->gfx.imu.funcs->init_microcode(adev);
 602                         if (err)
 603                                 dev_err(adev->dev, "Failed to load imu firmware!\n");
 604                 }
 605         }
 606
 607 out:
 608         if (err) {
 609                 amdgpu_ucode_release(&adev->gfx.pfp_fw);
 610                 amdgpu_ucode_release(&adev->gfx.me_fw);
 611                 amdgpu_ucode_release(&adev->gfx.rlc_fw);
 612                 amdgpu_ucode_release(&adev->gfx.mec_fw);
 613         }
 614
 615         return err;
 616 }
 617
 618 static u32 gfx_v12_0_get_csb_size(struct amdgpu_device *adev)
 619 {
 620         u32 count = 0;
 621         const struct cs_section_def *sect = NULL;
 622         const struct cs_extent_def *ext = NULL;
 623
 624         count += 1;
 625
 626         for (sect = gfx12_cs_data; sect->section != NULL; ++sect) {
 627                 if (sect->id == SECT_CONTEXT) {
 628                         for (ext = sect->section; ext->extent != NULL; ++ext)
 629                                 count += 2 + ext->reg_count;
 630                 } else
 631                         return 0;
 632         }
 633
 634         return count;
 635 }
 636
 637 static void gfx_v12_0_get_csb_buffer(struct amdgpu_device *adev,
 638                                      volatile u32 *buffer)
 639 {
 640         u32 count = 0, clustercount = 0, i;
 641         const struct cs_section_def *sect = NULL;
 642         const struct cs_extent_def *ext = NULL;
 643
 644         if (adev->gfx.rlc.cs_data == NULL)
 645                 return;
 646         if (buffer == NULL)
 647                 return;
 648
 649         count += 1;
 650
 651         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
 652                 if (sect->id == SECT_CONTEXT) {
 653                         for (ext = sect->section; ext->extent != NULL; ++ext) {
 654                                 clustercount++;
 655                                 buffer[count++] = ext->reg_count;
 656                                 buffer[count++] = ext->reg_index;
 657
 658                                 for (i = 0; i < ext->reg_count; i++)
 659                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
 660                         }
 661                 } else
 662                         return;
 663         }
 664
 665         buffer[0] = clustercount;
 666 }
 667
 668 static void gfx_v12_0_rlc_fini(struct amdgpu_device *adev)
 669 {
 670         /* clear state block */
 671         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
 672                         &adev->gfx.rlc.clear_state_gpu_addr,
 673                         (void **)&adev->gfx.rlc.cs_ptr);
 674
 675         /* jump table block */
 676         amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
 677                         &adev->gfx.rlc.cp_table_gpu_addr,
 678                         (void **)&adev->gfx.rlc.cp_table_ptr);
 679 }
 680
 681 static void gfx_v12_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
 682 {
 683         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
 684
 685         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
 686         reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 687         reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
 688         reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
 689         reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
 690         reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
 691         reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
 692         reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
 693         adev->gfx.rlc.rlcg_reg_access_supported = true;
 694 }
 695
 696 static int gfx_v12_0_rlc_init(struct amdgpu_device *adev)
 697 {
 698         const struct cs_section_def *cs_data;
 699         int r;
 700
 701         adev->gfx.rlc.cs_data = gfx12_cs_data;
 702
 703         cs_data = adev->gfx.rlc.cs_data;
 704
 705         if (cs_data) {
 706                 /* init clear state block */
 707                 r = amdgpu_gfx_rlc_init_csb(adev);
 708                 if (r)
 709                         return r;
 710         }
 711
 712         /* init spm vmid with 0xf */
 713         if (adev->gfx.rlc.funcs->update_spm_vmid)
 714                 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
 715
 716         return 0;
 717 }
 718
 719 static void gfx_v12_0_mec_fini(struct amdgpu_device *adev)
 720 {
 721         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
 722         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
 723         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
 724 }
 725
 726 static void gfx_v12_0_me_init(struct amdgpu_device *adev)
 727 {
 728         bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 729
 730         amdgpu_gfx_graphics_queue_acquire(adev);
 731 }
 732
 733 static int gfx_v12_0_mec_init(struct amdgpu_device *adev)
 734 {
 735         int r;
 736         u32 *hpd;
 737         size_t mec_hpd_size;
 738
 739         bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 740
 741         /* take ownership of the relevant compute queues */
 742         amdgpu_gfx_compute_queue_acquire(adev);
 743         mec_hpd_size = adev->gfx.num_compute_rings * GFX12_MEC_HPD_SIZE;
 744
 745         if (mec_hpd_size) {
 746                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
 747                                               AMDGPU_GEM_DOMAIN_GTT,
 748                                               &adev->gfx.mec.hpd_eop_obj,
 749                                               &adev->gfx.mec.hpd_eop_gpu_addr,
 750                                               (void **)&hpd);
 751                 if (r) {
 752                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
 753                         gfx_v12_0_mec_fini(adev);
 754                         return r;
 755                 }
 756
 757                 memset(hpd, 0, mec_hpd_size);
 758
 759                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
 760                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
 761         }
 762
 763         return 0;
 764 }
 765
 766 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
 767 {
 768         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 769                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 770                 (address << SQ_IND_INDEX__INDEX__SHIFT));
 771         return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 772 }
 773
 774 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
 775                            uint32_t thread, uint32_t regno,
 776                            uint32_t num, uint32_t *out)
 777 {
 778         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 779                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 780                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
 781                 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
 782                 (SQ_IND_INDEX__AUTO_INCR_MASK));
 783         while (num--)
 784                 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 785 }
 786
 787 static void gfx_v12_0_read_wave_data(struct amdgpu_device *adev,
 788                                      uint32_t xcc_id,
 789                                      uint32_t simd, uint32_t wave,
 790                                      uint32_t *dst, int *no_fields)
 791 {
 792         /* in gfx12 the SIMD_ID is specified as part of the INSTANCE
 793          * field when performing a select_se_sh so it should be
 794          * zero here */
 795         WARN_ON(simd != 0);
 796
 797         /* type 4 wave data */
 798         dst[(*no_fields)++] = 4;
 799         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
 800         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
 801         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
 802         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
 803         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
 804         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
 805         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
 806         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
 807         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
 808         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
 809         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
 810         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
 811         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
 812         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
 813         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATE_PRIV);
 814         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXCP_FLAG_PRIV);
 815         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXCP_FLAG_USER);
 816         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAP_CTRL);
 817         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_ACTIVE);
 818         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_VALID_AND_IDLE);
 819         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_DVGPR_ALLOC_LO);
 820         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_DVGPR_ALLOC_HI);
 821         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_SCHED_MODE);
 822 }
 823
 824 static void gfx_v12_0_read_wave_sgprs(struct amdgpu_device *adev,
 825                                       uint32_t xcc_id, uint32_t simd,
 826                                       uint32_t wave, uint32_t start,
 827                                       uint32_t size, uint32_t *dst)
 828 {
 829         WARN_ON(simd != 0);
 830
 831         wave_read_regs(
 832                 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
 833                 dst);
 834 }
 835
 836 static void gfx_v12_0_read_wave_vgprs(struct amdgpu_device *adev,
 837                                       uint32_t xcc_id, uint32_t simd,
 838                                       uint32_t wave, uint32_t thread,
 839                                       uint32_t start, uint32_t size,
 840                                       uint32_t *dst)
 841 {
 842         wave_read_regs(
 843                 adev, wave, thread,
 844                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
 845 }
 846
 847 static void gfx_v12_0_select_me_pipe_q(struct amdgpu_device *adev,
 848                                        u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
 849 {
 850         soc24_grbm_select(adev, me, pipe, q, vm);
 851 }
 852
 853 static const struct amdgpu_gfx_funcs gfx_v12_0_gfx_funcs = {
 854         .get_gpu_clock_counter = &gfx_v12_0_get_gpu_clock_counter,
 855         .select_se_sh = &gfx_v12_0_select_se_sh,
 856         .read_wave_data = &gfx_v12_0_read_wave_data,
 857         .read_wave_sgprs = &gfx_v12_0_read_wave_sgprs,
 858         .read_wave_vgprs = &gfx_v12_0_read_wave_vgprs,
 859         .select_me_pipe_q = &gfx_v12_0_select_me_pipe_q,
 860         .update_perfmon_mgcg = &gfx_v12_0_update_perf_clk,
 861 };
 862
 863 static int gfx_v12_0_gpu_early_init(struct amdgpu_device *adev)
 864 {
 865
 866         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 867         case IP_VERSION(12, 0, 0):
 868         case IP_VERSION(12, 0, 1):
 869                 adev->gfx.config.max_hw_contexts = 8;
 870                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
 871                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
 872                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
 873                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
 874                 break;
 875         default:
 876                 BUG();
 877                 break;
 878         }
 879
 880         return 0;
 881 }
 882
 883 static int gfx_v12_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
 884                                    int me, int pipe, int queue)
 885 {
 886         int r;
 887         struct amdgpu_ring *ring;
 888         unsigned int irq_type;
 889
 890         ring = &adev->gfx.gfx_ring[ring_id];
 891
 892         ring->me = me;
 893         ring->pipe = pipe;
 894         ring->queue = queue;
 895
 896         ring->ring_obj = NULL;
 897         ring->use_doorbell = true;
 898
 899         if (!ring_id)
 900                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
 901         else
 902                 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
 903         ring->vm_hub = AMDGPU_GFXHUB(0);
 904         sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 905
 906         irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
 907         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
 908                              AMDGPU_RING_PRIO_DEFAULT, NULL);
 909         if (r)
 910                 return r;
 911         return 0;
 912 }
 913
 914 static int gfx_v12_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
 915                                        int mec, int pipe, int queue)
 916 {
 917         int r;
 918         unsigned irq_type;
 919         struct amdgpu_ring *ring;
 920         unsigned int hw_prio;
 921
 922         ring = &adev->gfx.compute_ring[ring_id];
 923
 924         /* mec0 is me1 */
 925         ring->me = mec + 1;
 926         ring->pipe = pipe;
 927         ring->queue = queue;
 928
 929         ring->ring_obj = NULL;
 930         ring->use_doorbell = true;
 931         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
 932         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
 933                                 + (ring_id * GFX12_MEC_HPD_SIZE);
 934         ring->vm_hub = AMDGPU_GFXHUB(0);
 935         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 936
 937         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
 938                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
 939                 + ring->pipe;
 940         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
 941                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
 942         /* type-2 packets are deprecated on MEC, use type-3 instead */
 943         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
 944                              hw_prio, NULL);
 945         if (r)
 946                 return r;
 947
 948         return 0;
 949 }
 950
 951 static struct {
 952         SOC24_FIRMWARE_ID       id;
 953         unsigned int            offset;
 954         unsigned int            size;
 955         unsigned int            size_x16;
 956 } rlc_autoload_info[SOC24_FIRMWARE_ID_MAX];
 957
 958 #define RLC_TOC_OFFSET_DWUNIT   8
 959 #define RLC_SIZE_MULTIPLE       1024
 960 #define RLC_TOC_UMF_SIZE_inM    23ULL
 961 #define RLC_TOC_FORMAT_API      165ULL
 962
 963 static void gfx_v12_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
 964 {
 965         RLC_TABLE_OF_CONTENT_V2 *ucode = rlc_toc;
 966
 967         while (ucode && (ucode->id > SOC24_FIRMWARE_ID_INVALID)) {
 968                 rlc_autoload_info[ucode->id].id = ucode->id;
 969                 rlc_autoload_info[ucode->id].offset =
 970                         ucode->offset * RLC_TOC_OFFSET_DWUNIT * 4;
 971                 rlc_autoload_info[ucode->id].size =
 972                         ucode->size_x16 ? ucode->size * RLC_SIZE_MULTIPLE * 4 :
 973                                           ucode->size * 4;
 974                 ucode++;
 975         }
 976 }
 977
 978 static uint32_t gfx_v12_0_calc_toc_total_size(struct amdgpu_device *adev)
 979 {
 980         uint32_t total_size = 0;
 981         SOC24_FIRMWARE_ID id;
 982
 983         gfx_v12_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
 984
 985         for (id = SOC24_FIRMWARE_ID_RLC_G_UCODE; id < SOC24_FIRMWARE_ID_MAX; id++)
 986                 total_size += rlc_autoload_info[id].size;
 987
 988         /* In case the offset in rlc toc ucode is aligned */
 989         if (total_size < rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset)
 990                 total_size = rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset +
 991                         rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].size;
 992         if (total_size < (RLC_TOC_UMF_SIZE_inM << 20))
 993                 total_size = RLC_TOC_UMF_SIZE_inM << 20;
 994
 995         return total_size;
 996 }
 997
 998 static int gfx_v12_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
 999 {
1000         int r;
1001         uint32_t total_size;
1002
1003         total_size = gfx_v12_0_calc_toc_total_size(adev);
1004
1005         r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1006                                       AMDGPU_GEM_DOMAIN_VRAM,
1007                                       &adev->gfx.rlc.rlc_autoload_bo,
1008                                       &adev->gfx.rlc.rlc_autoload_gpu_addr,
1009                                       (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1010
1011         if (r) {
1012                 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1013                 return r;
1014         }
1015
1016         return 0;
1017 }
1018
1019 static void gfx_v12_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1020                                                        SOC24_FIRMWARE_ID id,
1021                                                        const void *fw_data,
1022                                                        uint32_t fw_size)
1023 {
1024         uint32_t toc_offset;
1025         uint32_t toc_fw_size;
1026         char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1027
1028         if (id <= SOC24_FIRMWARE_ID_INVALID || id >= SOC24_FIRMWARE_ID_MAX)
1029                 return;
1030
1031         toc_offset = rlc_autoload_info[id].offset;
1032         toc_fw_size = rlc_autoload_info[id].size;
1033
1034         if (fw_size == 0)
1035                 fw_size = toc_fw_size;
1036
1037         if (fw_size > toc_fw_size)
1038                 fw_size = toc_fw_size;
1039
1040         memcpy(ptr + toc_offset, fw_data, fw_size);
1041
1042         if (fw_size < toc_fw_size)
1043                 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
1044 }
1045
1046 static void
1047 gfx_v12_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev)
1048 {
1049         void *data;
1050         uint32_t size;
1051         uint32_t *toc_ptr;
1052
1053         data = adev->psp.toc.start_addr;
1054         size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_TOC].size;
1055
1056         toc_ptr = (uint32_t *)data + size / 4 - 2;
1057         *toc_ptr = (RLC_TOC_FORMAT_API << 24) | 0x1;
1058
1059         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_TOC,
1060                                                    data, size);
1061 }
1062
1063 static void
1064 gfx_v12_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev)
1065 {
1066         const __le32 *fw_data;
1067         uint32_t fw_size;
1068         const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1069         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1070         const struct rlc_firmware_header_v2_1 *rlcv21_hdr;
1071         const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1072         uint16_t version_major, version_minor;
1073
1074         /* pfp ucode */
1075         cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1076                 adev->gfx.pfp_fw->data;
1077         /* instruction */
1078         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1079                 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1080         fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1081         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_PFP,
1082                                                    fw_data, fw_size);
1083         /* data */
1084         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1085                 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1086         fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1087         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_PFP_P0_STACK,
1088                                                    fw_data, fw_size);
1089         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_PFP_P1_STACK,
1090                                                    fw_data, fw_size);
1091         /* me ucode */
1092         cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1093                 adev->gfx.me_fw->data;
1094         /* instruction */
1095         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1096                 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1097         fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1098         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_ME,
1099                                                    fw_data, fw_size);
1100         /* data */
1101         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1102                 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1103         fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1104         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_ME_P0_STACK,
1105                                                    fw_data, fw_size);
1106         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_ME_P1_STACK,
1107                                                    fw_data, fw_size);
1108         /* mec ucode */
1109         cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1110                 adev->gfx.mec_fw->data;
1111         /* instruction */
1112         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1113                 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1114         fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1115         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC,
1116                                                    fw_data, fw_size);
1117         /* data */
1118         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1119                 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1120         fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1121         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P0_STACK,
1122                                                    fw_data, fw_size);
1123         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P1_STACK,
1124                                                    fw_data, fw_size);
1125         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P2_STACK,
1126                                                    fw_data, fw_size);
1127         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P3_STACK,
1128                                                    fw_data, fw_size);
1129
1130         /* rlc ucode */
1131         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1132                 adev->gfx.rlc_fw->data;
1133         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1134                         le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1135         fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1136         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_G_UCODE,
1137                                                    fw_data, fw_size);
1138
1139         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1140         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1141         if (version_major == 2) {
1142                 if (version_minor >= 1) {
1143                         rlcv21_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1144
1145                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1146                                         le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_offset_bytes));
1147                         fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_size_bytes);
1148                         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLCG_SCRATCH,
1149                                                    fw_data, fw_size);
1150
1151                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1152                                         le32_to_cpu(rlcv21_hdr->save_restore_list_srm_offset_bytes));
1153                         fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_srm_size_bytes);
1154                         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_SRM_ARAM,
1155                                                    fw_data, fw_size);
1156                 }
1157                 if (version_minor >= 2) {
1158                         rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1159
1160                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1161                                         le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1162                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1163                         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_UCODE,
1164                                                    fw_data, fw_size);
1165
1166                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1167                                         le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1168                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1169                         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_DRAM_BOOT,
1170                                                    fw_data, fw_size);
1171                 }
1172         }
1173 }
1174
1175 static void
1176 gfx_v12_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev)
1177 {
1178         const __le32 *fw_data;
1179         uint32_t fw_size;
1180         const struct sdma_firmware_header_v3_0 *sdma_hdr;
1181
1182         sdma_hdr = (const struct sdma_firmware_header_v3_0 *)
1183                 adev->sdma.instance[0].fw->data;
1184         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1185                         le32_to_cpu(sdma_hdr->ucode_offset_bytes));
1186         fw_size = le32_to_cpu(sdma_hdr->ucode_size_bytes);
1187
1188         gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_SDMA_UCODE_TH0,
1189                                                    fw_data, fw_size);
1190 }
1191
1192 static void
1193 gfx_v12_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev)
1194 {
1195         const __le32 *fw_data;
1196         unsigned fw_size;
1197         const struct mes_firmware_header_v1_0 *mes_hdr;
1198         int pipe, ucode_id, data_id;
1199
1200         for (pipe = 0; pipe < 2; pipe++) {
1201                 if (pipe == 0) {
1202                         ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P0;
1203                         data_id  = SOC24_FIRMWARE_ID_RS64_MES_P0_STACK;
1204                 } else {
1205                         ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P1;
1206                         data_id  = SOC24_FIRMWARE_ID_RS64_MES_P1_STACK;
1207                 }
1208
1209                 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1210                         adev->mes.fw[pipe]->data;
1211
1212                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1213                                 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1214                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1215
1216                 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, ucode_id, fw_data, fw_size);
1217
1218                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1219                                 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1220                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1221
1222                 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, data_id, fw_data, fw_size);
1223         }
1224 }
1225
1226 static int gfx_v12_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1227 {
1228         uint32_t rlc_g_offset, rlc_g_size;
1229         uint64_t gpu_addr;
1230         uint32_t data;
1231
1232         /* RLC autoload sequence 2: copy ucode */
1233         gfx_v12_0_rlc_backdoor_autoload_copy_sdma_ucode(adev);
1234         gfx_v12_0_rlc_backdoor_autoload_copy_gfx_ucode(adev);
1235         gfx_v12_0_rlc_backdoor_autoload_copy_mes_ucode(adev);
1236         gfx_v12_0_rlc_backdoor_autoload_copy_toc_ucode(adev);
1237
1238         rlc_g_offset = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].offset;
1239         rlc_g_size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].size;
1240         gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset - adev->gmc.vram_start;
1241
1242         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1243         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1244
1245         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1246
1247         if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
1248                 /* RLC autoload sequence 3: load IMU fw */
1249                 if (adev->gfx.imu.funcs->load_microcode)
1250                         adev->gfx.imu.funcs->load_microcode(adev);
1251                 /* RLC autoload sequence 4 init IMU fw */
1252                 if (adev->gfx.imu.funcs->setup_imu)
1253                         adev->gfx.imu.funcs->setup_imu(adev);
1254                 if (adev->gfx.imu.funcs->start_imu)
1255                         adev->gfx.imu.funcs->start_imu(adev);
1256
1257                 /* RLC autoload sequence 5 disable gpa mode */
1258                 gfx_v12_0_disable_gpa_mode(adev);
1259         } else {
1260                 /* unhalt rlc to start autoload without imu */
1261                 data = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
1262                 data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD0_ENABLE, 1);
1263                 data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
1264                 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, data);
1265                 WREG32_SOC15(GC, 0, regRLC_CNTL, RLC_CNTL__RLC_ENABLE_F32_MASK);
1266         }
1267
1268         return 0;
1269 }
1270
1271 static void gfx_v12_0_alloc_ip_dump(struct amdgpu_device *adev)
1272 {
1273         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_0);
1274         uint32_t *ptr;
1275         uint32_t inst;
1276
1277         ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
1278         if (ptr == NULL) {
1279                 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
1280                 adev->gfx.ip_dump_core = NULL;
1281         } else {
1282                 adev->gfx.ip_dump_core = ptr;
1283         }
1284
1285         /* Allocate memory for compute queue registers for all the instances */
1286         reg_count = ARRAY_SIZE(gc_cp_reg_list_12);
1287         inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1288                 adev->gfx.mec.num_queue_per_pipe;
1289
1290         ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1291         if (ptr == NULL) {
1292                 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
1293                 adev->gfx.ip_dump_compute_queues = NULL;
1294         } else {
1295                 adev->gfx.ip_dump_compute_queues = ptr;
1296         }
1297
1298         /* Allocate memory for gfx queue registers for all the instances */
1299         reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_12);
1300         inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me *
1301                 adev->gfx.me.num_queue_per_pipe;
1302
1303         ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1304         if (ptr == NULL) {
1305                 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n");
1306                 adev->gfx.ip_dump_gfx_queues = NULL;
1307         } else {
1308                 adev->gfx.ip_dump_gfx_queues = ptr;
1309         }
1310 }
1311
1312 static int gfx_v12_0_sw_init(void *handle)
1313 {
1314         int i, j, k, r, ring_id = 0;
1315         unsigned num_compute_rings;
1316         int xcc_id = 0;
1317         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1318
1319         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1320         case IP_VERSION(12, 0, 0):
1321         case IP_VERSION(12, 0, 1):
1322                 adev->gfx.me.num_me = 1;
1323                 adev->gfx.me.num_pipe_per_me = 1;
1324                 adev->gfx.me.num_queue_per_pipe = 1;
1325                 adev->gfx.mec.num_mec = 2;
1326                 adev->gfx.mec.num_pipe_per_mec = 2;
1327                 adev->gfx.mec.num_queue_per_pipe = 4;
1328                 break;
1329         default:
1330                 adev->gfx.me.num_me = 1;
1331                 adev->gfx.me.num_pipe_per_me = 1;
1332                 adev->gfx.me.num_queue_per_pipe = 1;
1333                 adev->gfx.mec.num_mec = 1;
1334                 adev->gfx.mec.num_pipe_per_mec = 4;
1335                 adev->gfx.mec.num_queue_per_pipe = 8;
1336                 break;
1337         }
1338
1339         /* recalculate compute rings to use based on hardware configuration */
1340         num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
1341                              adev->gfx.mec.num_queue_per_pipe) / 2;
1342         adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
1343                                           num_compute_rings);
1344
1345         /* EOP Event */
1346         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1347                               GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1348                               &adev->gfx.eop_irq);
1349         if (r)
1350                 return r;
1351
1352         /* Privileged reg */
1353         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1354                               GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1355                               &adev->gfx.priv_reg_irq);
1356         if (r)
1357                 return r;
1358
1359         /* Privileged inst */
1360         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1361                               GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1362                               &adev->gfx.priv_inst_irq);
1363         if (r)
1364                 return r;
1365
1366         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1367
1368         gfx_v12_0_me_init(adev);
1369
1370         r = gfx_v12_0_rlc_init(adev);
1371         if (r) {
1372                 dev_err(adev->dev, "Failed to init rlc BOs!\n");
1373                 return r;
1374         }
1375
1376         r = gfx_v12_0_mec_init(adev);
1377         if (r) {
1378                 dev_err(adev->dev, "Failed to init MEC BOs!\n");
1379                 return r;
1380         }
1381
1382         /* set up the gfx ring */
1383         for (i = 0; i < adev->gfx.me.num_me; i++) {
1384                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1385                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1386                                 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1387                                         continue;
1388
1389                                 r = gfx_v12_0_gfx_ring_init(adev, ring_id,
1390                                                             i, k, j);
1391                                 if (r)
1392                                         return r;
1393                                 ring_id++;
1394                         }
1395                 }
1396         }
1397
1398         ring_id = 0;
1399         /* set up the compute queues - allocate horizontally across pipes */
1400         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1401                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1402                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1403                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev,
1404                                                                 0, i, k, j))
1405                                         continue;
1406
1407                                 r = gfx_v12_0_compute_ring_init(adev, ring_id,
1408                                                                 i, k, j);
1409                                 if (r)
1410                                         return r;
1411
1412                                 ring_id++;
1413                         }
1414                 }
1415         }
1416
1417         if (!adev->enable_mes_kiq) {
1418                 r = amdgpu_gfx_kiq_init(adev, GFX12_MEC_HPD_SIZE, 0);
1419                 if (r) {
1420                         dev_err(adev->dev, "Failed to init KIQ BOs!\n");
1421                         return r;
1422                 }
1423
1424                 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1425                 if (r)
1426                         return r;
1427         }
1428
1429         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v12_compute_mqd), 0);
1430         if (r)
1431                 return r;
1432
1433         /* allocate visible FB for rlc auto-loading fw */
1434         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1435                 r = gfx_v12_0_rlc_autoload_buffer_init(adev);
1436                 if (r)
1437                         return r;
1438         }
1439
1440         r = gfx_v12_0_gpu_early_init(adev);
1441         if (r)
1442                 return r;
1443
1444         gfx_v12_0_alloc_ip_dump(adev);
1445
1446         return 0;
1447 }
1448
1449 static void gfx_v12_0_pfp_fini(struct amdgpu_device *adev)
1450 {
1451         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1452                               &adev->gfx.pfp.pfp_fw_gpu_addr,
1453                               (void **)&adev->gfx.pfp.pfp_fw_ptr);
1454
1455         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1456                               &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1457                               (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1458 }
1459
1460 static void gfx_v12_0_me_fini(struct amdgpu_device *adev)
1461 {
1462         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1463                               &adev->gfx.me.me_fw_gpu_addr,
1464                               (void **)&adev->gfx.me.me_fw_ptr);
1465
1466         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1467                                &adev->gfx.me.me_fw_data_gpu_addr,
1468                                (void **)&adev->gfx.me.me_fw_data_ptr);
1469 }
1470
1471 static void gfx_v12_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1472 {
1473         amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1474                         &adev->gfx.rlc.rlc_autoload_gpu_addr,
1475                         (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1476 }
1477
1478 static int gfx_v12_0_sw_fini(void *handle)
1479 {
1480         int i;
1481         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1482
1483         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1484                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1485         for (i = 0; i < adev->gfx.num_compute_rings; i++)
1486                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1487
1488         amdgpu_gfx_mqd_sw_fini(adev, 0);
1489
1490         if (!adev->enable_mes_kiq) {
1491                 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
1492                 amdgpu_gfx_kiq_fini(adev, 0);
1493         }
1494
1495         gfx_v12_0_pfp_fini(adev);
1496         gfx_v12_0_me_fini(adev);
1497         gfx_v12_0_rlc_fini(adev);
1498         gfx_v12_0_mec_fini(adev);
1499
1500         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1501                 gfx_v12_0_rlc_autoload_buffer_fini(adev);
1502
1503         gfx_v12_0_free_microcode(adev);
1504
1505         kfree(adev->gfx.ip_dump_core);
1506         kfree(adev->gfx.ip_dump_compute_queues);
1507         kfree(adev->gfx.ip_dump_gfx_queues);
1508
1509         return 0;
1510 }
1511
1512 static void gfx_v12_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1513                                    u32 sh_num, u32 instance, int xcc_id)
1514 {
1515         u32 data;
1516
1517         if (instance == 0xffffffff)
1518                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1519                                      INSTANCE_BROADCAST_WRITES, 1);
1520         else
1521                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1522                                      instance);
1523
1524         if (se_num == 0xffffffff)
1525                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1526                                      1);
1527         else
1528                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1529
1530         if (sh_num == 0xffffffff)
1531                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1532                                      1);
1533         else
1534                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1535
1536         WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
1537 }
1538
1539 static u32 gfx_v12_0_get_sa_active_bitmap(struct amdgpu_device *adev)
1540 {
1541         u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1542
1543         gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regGRBM_CC_GC_SA_UNIT_DISABLE);
1544         gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1545                                             GRBM_CC_GC_SA_UNIT_DISABLE,
1546                                             SA_DISABLE);
1547         gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGRBM_GC_USER_SA_UNIT_DISABLE);
1548         gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1549                                                  GRBM_GC_USER_SA_UNIT_DISABLE,
1550                                                  SA_DISABLE);
1551         sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1552                                             adev->gfx.config.max_shader_engines);
1553
1554         return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1555 }
1556
1557 static u32 gfx_v12_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1558 {
1559         u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1560         u32 rb_mask;
1561
1562         gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
1563         gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1564                                             CC_RB_BACKEND_DISABLE,
1565                                             BACKEND_DISABLE);
1566         gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
1567         gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
1568                                                  GC_USER_RB_BACKEND_DISABLE,
1569                                                  BACKEND_DISABLE);
1570         rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1571                                             adev->gfx.config.max_shader_engines);
1572
1573         return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
1574 }
1575
1576 static void gfx_v12_0_setup_rb(struct amdgpu_device *adev)
1577 {
1578         u32 rb_bitmap_width_per_sa;
1579         u32 max_sa;
1580         u32 active_sa_bitmap;
1581         u32 global_active_rb_bitmap;
1582         u32 active_rb_bitmap = 0;
1583         u32 i;
1584
1585         /* query sa bitmap from SA_UNIT_DISABLE registers */
1586         active_sa_bitmap = gfx_v12_0_get_sa_active_bitmap(adev);
1587         /* query rb bitmap from RB_BACKEND_DISABLE registers */
1588         global_active_rb_bitmap = gfx_v12_0_get_rb_active_bitmap(adev);
1589
1590         /* generate active rb bitmap according to active sa bitmap */
1591         max_sa = adev->gfx.config.max_shader_engines *
1592                  adev->gfx.config.max_sh_per_se;
1593         rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1594                                  adev->gfx.config.max_sh_per_se;
1595         for (i = 0; i < max_sa; i++) {
1596                 if (active_sa_bitmap & (1 << i))
1597                         active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
1598         }
1599
1600         active_rb_bitmap |= global_active_rb_bitmap;
1601         adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1602         adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1603 }
1604
1605 #define LDS_APP_BASE           0x1
1606 #define SCRATCH_APP_BASE       0x2
1607
1608 static void gfx_v12_0_init_compute_vmid(struct amdgpu_device *adev)
1609 {
1610         int i;
1611         uint32_t sh_mem_bases;
1612         uint32_t data;
1613
1614         /*
1615          * Configure apertures:
1616          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1617          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1618          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1619          */
1620         sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
1621                         SCRATCH_APP_BASE;
1622
1623         mutex_lock(&adev->srbm_mutex);
1624         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1625                 soc24_grbm_select(adev, 0, 0, 0, i);
1626                 /* CP and shaders */
1627                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1628                 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
1629
1630                 /* Enable trap for each kfd vmid. */
1631                 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL);
1632                 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1633                 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data);
1634         }
1635         soc24_grbm_select(adev, 0, 0, 0, 0);
1636         mutex_unlock(&adev->srbm_mutex);
1637 }
1638
1639 static void gfx_v12_0_tcp_harvest(struct amdgpu_device *adev)
1640 {
1641         /* TODO: harvest feature to be added later. */
1642 }
1643
1644 static void gfx_v12_0_get_tcc_info(struct amdgpu_device *adev)
1645 {
1646 }
1647
1648 static void gfx_v12_0_constants_init(struct amdgpu_device *adev)
1649 {
1650         u32 tmp;
1651         int i;
1652
1653         if (!amdgpu_sriov_vf(adev))
1654                 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1655
1656         gfx_v12_0_setup_rb(adev);
1657         gfx_v12_0_get_cu_info(adev, &adev->gfx.cu_info);
1658         gfx_v12_0_get_tcc_info(adev);
1659         adev->gfx.config.pa_sc_tile_steering_override = 0;
1660
1661         /* XXX SH_MEM regs */
1662         /* where to put LDS, scratch, GPUVM in FSA64 space */
1663         mutex_lock(&adev->srbm_mutex);
1664         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1665                 soc24_grbm_select(adev, 0, 0, 0, i);
1666                 /* CP and shaders */
1667                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1668                 if (i != 0) {
1669                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1670                                 (adev->gmc.private_aperture_start >> 48));
1671                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1672                                 (adev->gmc.shared_aperture_start >> 48));
1673                         WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
1674                 }
1675         }
1676         soc24_grbm_select(adev, 0, 0, 0, 0);
1677
1678         mutex_unlock(&adev->srbm_mutex);
1679
1680         gfx_v12_0_init_compute_vmid(adev);
1681 }
1682
1683 static void gfx_v12_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1684                                                 bool enable)
1685 {
1686         u32 tmp;
1687
1688         if (amdgpu_sriov_vf(adev))
1689                 return;
1690
1691         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0);
1692
1693         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
1694                             enable ? 1 : 0);
1695         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
1696                             enable ? 1 : 0);
1697         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
1698                             enable ? 1 : 0);
1699         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
1700                             enable ? 1 : 0);
1701
1702         WREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0, tmp);
1703 }
1704
1705 static int gfx_v12_0_init_csb(struct amdgpu_device *adev)
1706 {
1707         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
1708
1709         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
1710                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
1711         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
1712                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1713         WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
1714
1715         return 0;
1716 }
1717
1718 static void gfx_v12_0_rlc_stop(struct amdgpu_device *adev)
1719 {
1720         u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
1721
1722         tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1723         WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
1724 }
1725
1726 static void gfx_v12_0_rlc_reset(struct amdgpu_device *adev)
1727 {
1728         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1729         udelay(50);
1730         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
1731         udelay(50);
1732 }
1733
1734 static void gfx_v12_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
1735                                              bool enable)
1736 {
1737         uint32_t rlc_pg_cntl;
1738
1739         rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
1740
1741         if (!enable) {
1742                 /* RLC_PG_CNTL[23] = 0 (default)
1743                  * RLC will wait for handshake acks with SMU
1744                  * GFXOFF will be enabled
1745                  * RLC_PG_CNTL[23] = 1
1746                  * RLC will not issue any message to SMU
1747                  * hence no handshake between SMU & RLC
1748                  * GFXOFF will be disabled
1749                  */
1750                 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1751         } else
1752                 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1753         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
1754 }
1755
1756 static void gfx_v12_0_rlc_start(struct amdgpu_device *adev)
1757 {
1758         /* TODO: enable rlc & smu handshake until smu
1759          * and gfxoff feature works as expected */
1760         if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
1761                 gfx_v12_0_rlc_smu_handshake_cntl(adev, false);
1762
1763         WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
1764         udelay(50);
1765 }
1766
1767 static void gfx_v12_0_rlc_enable_srm(struct amdgpu_device *adev)
1768 {
1769         uint32_t tmp;
1770
1771         /* enable Save Restore Machine */
1772         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
1773         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
1774         tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
1775         WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
1776 }
1777
1778 static void gfx_v12_0_load_rlcg_microcode(struct amdgpu_device *adev)
1779 {
1780         const struct rlc_firmware_header_v2_0 *hdr;
1781         const __le32 *fw_data;
1782         unsigned i, fw_size;
1783
1784         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1785         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1786                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1787         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1788
1789         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
1790                      RLCG_UCODE_LOADING_START_ADDRESS);
1791
1792         for (i = 0; i < fw_size; i++)
1793                 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
1794                              le32_to_cpup(fw_data++));
1795
1796         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
1797 }
1798
1799 static void gfx_v12_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
1800 {
1801         const struct rlc_firmware_header_v2_2 *hdr;
1802         const __le32 *fw_data;
1803         unsigned i, fw_size;
1804         u32 tmp;
1805
1806         hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1807
1808         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1809                         le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
1810         fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
1811
1812         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
1813
1814         for (i = 0; i < fw_size; i++) {
1815                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1816                         msleep(1);
1817                 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
1818                                 le32_to_cpup(fw_data++));
1819         }
1820
1821         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1822
1823         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1824                         le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
1825         fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
1826
1827         WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
1828         for (i = 0; i < fw_size; i++) {
1829                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1830                         msleep(1);
1831                 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
1832                                 le32_to_cpup(fw_data++));
1833         }
1834
1835         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1836
1837         tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
1838         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
1839         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
1840         WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
1841 }
1842
1843 static int gfx_v12_0_rlc_load_microcode(struct amdgpu_device *adev)
1844 {
1845         const struct rlc_firmware_header_v2_0 *hdr;
1846         uint16_t version_major;
1847         uint16_t version_minor;
1848
1849         if (!adev->gfx.rlc_fw)
1850                 return -EINVAL;
1851
1852         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1853         amdgpu_ucode_print_rlc_hdr(&hdr->header);
1854
1855         version_major = le16_to_cpu(hdr->header.header_version_major);
1856         version_minor = le16_to_cpu(hdr->header.header_version_minor);
1857
1858         if (version_major == 2) {
1859                 gfx_v12_0_load_rlcg_microcode(adev);
1860                 if (amdgpu_dpm == 1) {
1861                         if (version_minor >= 2)
1862                                 gfx_v12_0_load_rlc_iram_dram_microcode(adev);
1863                 }
1864
1865                 return 0;
1866         }
1867
1868         return -EINVAL;
1869 }
1870
1871 static int gfx_v12_0_rlc_resume(struct amdgpu_device *adev)
1872 {
1873         int r;
1874
1875         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1876                 gfx_v12_0_init_csb(adev);
1877
1878                 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
1879                         gfx_v12_0_rlc_enable_srm(adev);
1880         } else {
1881                 if (amdgpu_sriov_vf(adev)) {
1882                         gfx_v12_0_init_csb(adev);
1883                         return 0;
1884                 }
1885
1886                 adev->gfx.rlc.funcs->stop(adev);
1887
1888                 /* disable CG */
1889                 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
1890
1891                 /* disable PG */
1892                 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
1893
1894                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1895                         /* legacy rlc firmware loading */
1896                         r = gfx_v12_0_rlc_load_microcode(adev);
1897                         if (r)
1898                                 return r;
1899                 }
1900
1901                 gfx_v12_0_init_csb(adev);
1902
1903                 adev->gfx.rlc.funcs->start(adev);
1904         }
1905
1906         return 0;
1907 }
1908
1909 static void gfx_v12_0_config_gfx_rs64(struct amdgpu_device *adev)
1910 {
1911         const struct gfx_firmware_header_v2_0 *pfp_hdr;
1912         const struct gfx_firmware_header_v2_0 *me_hdr;
1913         const struct gfx_firmware_header_v2_0 *mec_hdr;
1914         uint32_t pipe_id, tmp;
1915
1916         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
1917                 adev->gfx.mec_fw->data;
1918         me_hdr = (const struct gfx_firmware_header_v2_0 *)
1919                 adev->gfx.me_fw->data;
1920         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
1921                 adev->gfx.pfp_fw->data;
1922
1923         /* config pfp program start addr */
1924         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
1925                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
1926                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
1927                         (pfp_hdr->ucode_start_addr_hi << 30) |
1928                         (pfp_hdr->ucode_start_addr_lo >> 2));
1929                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
1930                         pfp_hdr->ucode_start_addr_hi >> 2);
1931         }
1932         soc24_grbm_select(adev, 0, 0, 0, 0);
1933
1934         /* reset pfp pipe */
1935         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
1936         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
1937         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
1938         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1939
1940         /* clear pfp pipe reset */
1941         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
1942         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
1943         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1944
1945         /* config me program start addr */
1946         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
1947                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
1948                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
1949                         (me_hdr->ucode_start_addr_hi << 30) |
1950                         (me_hdr->ucode_start_addr_lo >> 2));
1951                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
1952                         me_hdr->ucode_start_addr_hi>>2);
1953         }
1954         soc24_grbm_select(adev, 0, 0, 0, 0);
1955
1956         /* reset me pipe */
1957         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
1958         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
1959         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
1960         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1961
1962         /* clear me pipe reset */
1963         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
1964         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
1965         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
1966
1967         /* config mec program start addr */
1968         for (pipe_id = 0; pipe_id < 4; pipe_id++) {
1969                 soc24_grbm_select(adev, 1, pipe_id, 0, 0);
1970                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
1971                                         mec_hdr->ucode_start_addr_lo >> 2 |
1972                                         mec_hdr->ucode_start_addr_hi << 30);
1973                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
1974                                         mec_hdr->ucode_start_addr_hi >> 2);
1975         }
1976         soc24_grbm_select(adev, 0, 0, 0, 0);
1977
1978         /* reset mec pipe */
1979         tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
1980         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
1981         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
1982         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
1983         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
1984         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
1985
1986         /* clear mec pipe reset */
1987         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
1988         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
1989         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
1990         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
1991         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
1992 }
1993
1994 static void gfx_v12_0_set_pfp_ucode_start_addr(struct amdgpu_device *adev)
1995 {
1996         const struct gfx_firmware_header_v2_0 *cp_hdr;
1997         unsigned pipe_id, tmp;
1998
1999         cp_hdr = (const struct gfx_firmware_header_v2_0 *)
2000                 adev->gfx.pfp_fw->data;
2001         mutex_lock(&adev->srbm_mutex);
2002         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2003                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2004                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2005                              (cp_hdr->ucode_start_addr_hi << 30) |
2006                              (cp_hdr->ucode_start_addr_lo >> 2));
2007                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2008                              cp_hdr->ucode_start_addr_hi>>2);
2009
2010                 /*
2011                  * Program CP_ME_CNTL to reset given PIPE to take
2012                  * effect of CP_PFP_PRGRM_CNTR_START.
2013                  */
2014                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2015                 if (pipe_id == 0)
2016                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2017                                         PFP_PIPE0_RESET, 1);
2018                 else
2019                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2020                                         PFP_PIPE1_RESET, 1);
2021                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2022
2023                 /* Clear pfp pipe0 reset bit. */
2024                 if (pipe_id == 0)
2025                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2026                                         PFP_PIPE0_RESET, 0);
2027                 else
2028                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2029                                         PFP_PIPE1_RESET, 0);
2030                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2031         }
2032         soc24_grbm_select(adev, 0, 0, 0, 0);
2033         mutex_unlock(&adev->srbm_mutex);
2034 }
2035
2036 static void gfx_v12_0_set_me_ucode_start_addr(struct amdgpu_device *adev)
2037 {
2038         const struct gfx_firmware_header_v2_0 *cp_hdr;
2039         unsigned pipe_id, tmp;
2040
2041         cp_hdr = (const struct gfx_firmware_header_v2_0 *)
2042                 adev->gfx.me_fw->data;
2043         mutex_lock(&adev->srbm_mutex);
2044         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2045                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2046                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2047                              (cp_hdr->ucode_start_addr_hi << 30) |
2048                              (cp_hdr->ucode_start_addr_lo >> 2) );
2049                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2050                              cp_hdr->ucode_start_addr_hi>>2);
2051
2052                 /*
2053                  * Program CP_ME_CNTL to reset given PIPE to take
2054                  * effect of CP_ME_PRGRM_CNTR_START.
2055                  */
2056                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2057                 if (pipe_id == 0)
2058                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2059                                         ME_PIPE0_RESET, 1);
2060                 else
2061                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2062                                         ME_PIPE1_RESET, 1);
2063                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2064
2065                 /* Clear pfp pipe0 reset bit. */
2066                 if (pipe_id == 0)
2067                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2068                                         ME_PIPE0_RESET, 0);
2069                 else
2070                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2071                                         ME_PIPE1_RESET, 0);
2072                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2073         }
2074         soc24_grbm_select(adev, 0, 0, 0, 0);
2075         mutex_unlock(&adev->srbm_mutex);
2076 }
2077
2078 static void gfx_v12_0_set_mec_ucode_start_addr(struct amdgpu_device *adev)
2079 {
2080         const struct gfx_firmware_header_v2_0 *cp_hdr;
2081         unsigned pipe_id;
2082
2083         cp_hdr = (const struct gfx_firmware_header_v2_0 *)
2084                 adev->gfx.mec_fw->data;
2085         mutex_lock(&adev->srbm_mutex);
2086         for (pipe_id = 0; pipe_id < adev->gfx.mec.num_pipe_per_mec; pipe_id++) {
2087                 soc24_grbm_select(adev, 1, pipe_id, 0, 0);
2088                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2089                              cp_hdr->ucode_start_addr_lo >> 2 |
2090                              cp_hdr->ucode_start_addr_hi << 30);
2091                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2092                              cp_hdr->ucode_start_addr_hi >> 2);
2093         }
2094         soc24_grbm_select(adev, 0, 0, 0, 0);
2095         mutex_unlock(&adev->srbm_mutex);
2096 }
2097
2098 static int gfx_v12_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2099 {
2100         uint32_t cp_status;
2101         uint32_t bootload_status;
2102         int i;
2103
2104         for (i = 0; i < adev->usec_timeout; i++) {
2105                 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
2106                 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
2107
2108                 if ((cp_status == 0) &&
2109                     (REG_GET_FIELD(bootload_status,
2110                         RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
2111                         break;
2112                 }
2113                 udelay(1);
2114                 if (amdgpu_emu_mode)
2115                         msleep(10);
2116         }
2117
2118         if (i >= adev->usec_timeout) {
2119                 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
2120                 return -ETIMEDOUT;
2121         }
2122
2123         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2124                 gfx_v12_0_set_pfp_ucode_start_addr(adev);
2125                 gfx_v12_0_set_me_ucode_start_addr(adev);
2126                 gfx_v12_0_set_mec_ucode_start_addr(adev);
2127         }
2128
2129         return 0;
2130 }
2131
2132 static int gfx_v12_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2133 {
2134         int i;
2135         u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2136
2137         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2138         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2139         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2140
2141         for (i = 0; i < adev->usec_timeout; i++) {
2142                 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
2143                         break;
2144                 udelay(1);
2145         }
2146
2147         if (i >= adev->usec_timeout)
2148                 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
2149
2150         return 0;
2151 }
2152
2153 static int gfx_v12_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
2154 {
2155         int r;
2156         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2157         const __le32 *fw_ucode, *fw_data;
2158         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
2159         uint32_t tmp;
2160         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2161
2162         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2163                 adev->gfx.pfp_fw->data;
2164
2165         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2166
2167         /* instruction */
2168         fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
2169                 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
2170         fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
2171         /* data */
2172         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2173                 le32_to_cpu(pfp_hdr->data_offset_bytes));
2174         fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
2175
2176         /* 64kb align */
2177         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2178                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2179                                       &adev->gfx.pfp.pfp_fw_obj,
2180                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
2181                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
2182         if (r) {
2183                 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
2184                 gfx_v12_0_pfp_fini(adev);
2185                 return r;
2186         }
2187
2188         r = amdgpu_bo_create_reserved(adev, fw_data_size,
2189                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2190                                       &adev->gfx.pfp.pfp_fw_data_obj,
2191                                       &adev->gfx.pfp.pfp_fw_data_gpu_addr,
2192                                       (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
2193         if (r) {
2194                 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
2195                 gfx_v12_0_pfp_fini(adev);
2196                 return r;
2197         }
2198
2199         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
2200         memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
2201
2202         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2203         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
2204         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2205         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
2206
2207         if (amdgpu_emu_mode == 1)
2208                 adev->hdp.funcs->flush_hdp(adev, NULL);
2209
2210         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2211                 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2212         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2213                 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2214
2215         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2216         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2217         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2218         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2219         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2220
2221         /*
2222          * Programming any of the CP_PFP_IC_BASE registers
2223          * forces invalidation of the ME L1 I$. Wait for the
2224          * invalidation complete
2225          */
2226         for (i = 0; i < usec_timeout; i++) {
2227                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2228                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2229                         INVALIDATE_CACHE_COMPLETE))
2230                         break;
2231                 udelay(1);
2232         }
2233
2234         if (i >= usec_timeout) {
2235                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2236                 return -EINVAL;
2237         }
2238
2239         /* Prime the L1 instruction caches */
2240         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2241         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2242         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2243         /* Waiting for cache primed*/
2244         for (i = 0; i < usec_timeout; i++) {
2245                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2246                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2247                         ICACHE_PRIMED))
2248                         break;
2249                 udelay(1);
2250         }
2251
2252         if (i >= usec_timeout) {
2253                 dev_err(adev->dev, "failed to prime instruction cache\n");
2254                 return -EINVAL;
2255         }
2256
2257         mutex_lock(&adev->srbm_mutex);
2258         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2259                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2260
2261                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2262                         lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
2263                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2264                         upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
2265         }
2266         soc24_grbm_select(adev, 0, 0, 0, 0);
2267         mutex_unlock(&adev->srbm_mutex);
2268
2269         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2270         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2271         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2272         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2273
2274         /* Invalidate the data caches */
2275         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2276         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2277         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2278
2279         for (i = 0; i < usec_timeout; i++) {
2280                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2281                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2282                         INVALIDATE_DCACHE_COMPLETE))
2283                         break;
2284                 udelay(1);
2285         }
2286
2287         if (i >= usec_timeout) {
2288                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2289                 return -EINVAL;
2290         }
2291
2292         gfx_v12_0_set_pfp_ucode_start_addr(adev);
2293
2294         return 0;
2295 }
2296
2297 static int gfx_v12_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
2298 {
2299         int r;
2300         const struct gfx_firmware_header_v2_0 *me_hdr;
2301         const __le32 *fw_ucode, *fw_data;
2302         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
2303         uint32_t tmp;
2304         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2305
2306         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2307                 adev->gfx.me_fw->data;
2308
2309         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2310
2311         /* instruction */
2312         fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
2313                 le32_to_cpu(me_hdr->ucode_offset_bytes));
2314         fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
2315         /* data */
2316         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
2317                 le32_to_cpu(me_hdr->data_offset_bytes));
2318         fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
2319
2320         /* 64kb align*/
2321         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2322                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2323                                       &adev->gfx.me.me_fw_obj,
2324                                       &adev->gfx.me.me_fw_gpu_addr,
2325                                       (void **)&adev->gfx.me.me_fw_ptr);
2326         if (r) {
2327                 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
2328                 gfx_v12_0_me_fini(adev);
2329                 return r;
2330         }
2331
2332         r = amdgpu_bo_create_reserved(adev, fw_data_size,
2333                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2334                                       &adev->gfx.me.me_fw_data_obj,
2335                                       &adev->gfx.me.me_fw_data_gpu_addr,
2336                                       (void **)&adev->gfx.me.me_fw_data_ptr);
2337         if (r) {
2338                 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
2339                 gfx_v12_0_pfp_fini(adev);
2340                 return r;
2341         }
2342
2343         memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
2344         memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
2345
2346         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
2347         amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
2348         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
2349         amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
2350
2351         if (amdgpu_emu_mode == 1)
2352                 adev->hdp.funcs->flush_hdp(adev, NULL);
2353
2354         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2355                 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
2356         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2357                 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
2358
2359         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2360         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2361         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2362         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2363         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2364
2365         /*
2366          * Programming any of the CP_ME_IC_BASE registers
2367          * forces invalidation of the ME L1 I$. Wait for the
2368          * invalidation complete
2369          */
2370         for (i = 0; i < usec_timeout; i++) {
2371                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2372                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2373                         INVALIDATE_CACHE_COMPLETE))
2374                         break;
2375                 udelay(1);
2376         }
2377
2378         if (i >= usec_timeout) {
2379                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2380                 return -EINVAL;
2381         }
2382
2383         /* Prime the instruction caches */
2384         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2385         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2386         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2387
2388         /* Waiting for instruction cache primed*/
2389         for (i = 0; i < usec_timeout; i++) {
2390                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2391                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2392                         ICACHE_PRIMED))
2393                         break;
2394                 udelay(1);
2395         }
2396
2397         if (i >= usec_timeout) {
2398                 dev_err(adev->dev, "failed to prime instruction cache\n");
2399                 return -EINVAL;
2400         }
2401
2402         mutex_lock(&adev->srbm_mutex);
2403         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2404                 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2405
2406                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2407                         lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
2408                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2409                         upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
2410         }
2411         soc24_grbm_select(adev, 0, 0, 0, 0);
2412         mutex_unlock(&adev->srbm_mutex);
2413
2414         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2415         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2416         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2417         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2418
2419         /* Invalidate the data caches */
2420         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2421         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2422         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2423
2424         for (i = 0; i < usec_timeout; i++) {
2425                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2426                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2427                         INVALIDATE_DCACHE_COMPLETE))
2428                         break;
2429                 udelay(1);
2430         }
2431
2432         if (i >= usec_timeout) {
2433                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2434                 return -EINVAL;
2435         }
2436
2437         gfx_v12_0_set_me_ucode_start_addr(adev);
2438
2439         return 0;
2440 }
2441
2442 static int gfx_v12_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2443 {
2444         int r;
2445
2446         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
2447                 return -EINVAL;
2448
2449         gfx_v12_0_cp_gfx_enable(adev, false);
2450
2451         r = gfx_v12_0_cp_gfx_load_pfp_microcode_rs64(adev);
2452         if (r) {
2453                 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
2454                 return r;
2455         }
2456
2457         r = gfx_v12_0_cp_gfx_load_me_microcode_rs64(adev);
2458         if (r) {
2459                 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
2460                 return r;
2461         }
2462
2463         return 0;
2464 }
2465
2466 static int gfx_v12_0_cp_gfx_start(struct amdgpu_device *adev)
2467 {
2468         /* init the CP */
2469         WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
2470                      adev->gfx.config.max_hw_contexts - 1);
2471         WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
2472
2473         if (!amdgpu_async_gfx_ring)
2474                 gfx_v12_0_cp_gfx_enable(adev, true);
2475
2476         return 0;
2477 }
2478
2479 static void gfx_v12_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
2480                                          CP_PIPE_ID pipe)
2481 {
2482         u32 tmp;
2483
2484         tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
2485         tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
2486
2487         WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
2488 }
2489
2490 static void gfx_v12_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
2491                                           struct amdgpu_ring *ring)
2492 {
2493         u32 tmp;
2494
2495         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
2496         if (ring->use_doorbell) {
2497                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2498                                     DOORBELL_OFFSET, ring->doorbell_index);
2499                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2500                                     DOORBELL_EN, 1);
2501         } else {
2502                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2503                                     DOORBELL_EN, 0);
2504         }
2505         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
2506
2507         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
2508                             DOORBELL_RANGE_LOWER, ring->doorbell_index);
2509         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
2510
2511         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
2512                      CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
2513 }
2514
2515 static int gfx_v12_0_cp_gfx_resume(struct amdgpu_device *adev)
2516 {
2517         struct amdgpu_ring *ring;
2518         u32 tmp;
2519         u32 rb_bufsz;
2520         u64 rb_addr, rptr_addr, wptr_gpu_addr;
2521         u32 i;
2522
2523         /* Set the write pointer delay */
2524         WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
2525
2526         /* set the RB to use vmid 0 */
2527         WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
2528
2529         /* Init gfx ring 0 for pipe 0 */
2530         mutex_lock(&adev->srbm_mutex);
2531         gfx_v12_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
2532
2533         /* Set ring buffer size */
2534         ring = &adev->gfx.gfx_ring[0];
2535         rb_bufsz = order_base_2(ring->ring_size / 8);
2536         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
2537         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
2538         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
2539
2540         /* Initialize the ring buffer's write pointers */
2541         ring->wptr = 0;
2542         WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
2543         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
2544
2545         /* set the wb address wether it's enabled or not */
2546         rptr_addr = ring->rptr_gpu_addr;
2547         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
2548         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
2549                      CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
2550
2551         wptr_gpu_addr = ring->wptr_gpu_addr;
2552         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
2553                      lower_32_bits(wptr_gpu_addr));
2554         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
2555                      upper_32_bits(wptr_gpu_addr));
2556
2557         mdelay(1);
2558         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
2559
2560         rb_addr = ring->gpu_addr >> 8;
2561         WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
2562         WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
2563
2564         WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
2565
2566         gfx_v12_0_cp_gfx_set_doorbell(adev, ring);
2567         mutex_unlock(&adev->srbm_mutex);
2568
2569         /* Switch to pipe 0 */
2570         mutex_lock(&adev->srbm_mutex);
2571         gfx_v12_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
2572         mutex_unlock(&adev->srbm_mutex);
2573
2574         /* start the ring */
2575         gfx_v12_0_cp_gfx_start(adev);
2576
2577         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2578                 ring = &adev->gfx.gfx_ring[i];
2579                 ring->sched.ready = true;
2580         }
2581
2582         return 0;
2583 }
2584
2585 static void gfx_v12_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
2586 {
2587         u32 data;
2588
2589         data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
2590         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
2591                                                  enable ? 0 : 1);
2592         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
2593                                                  enable ? 0 : 1);
2594         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
2595                                                  enable ? 0 : 1);
2596         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
2597                                                  enable ? 0 : 1);
2598         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
2599                                                  enable ? 0 : 1);
2600         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
2601                                                  enable ? 1 : 0);
2602         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
2603                                                  enable ? 1 : 0);
2604         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
2605                                                  enable ? 1 : 0);
2606         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
2607                                                  enable ? 1 : 0);
2608         data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
2609                                                  enable ? 0 : 1);
2610         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
2611
2612         adev->gfx.kiq[0].ring.sched.ready = enable;
2613
2614         udelay(50);
2615 }
2616
2617 static int gfx_v12_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
2618 {
2619         const struct gfx_firmware_header_v2_0 *mec_hdr;
2620         const __le32 *fw_ucode, *fw_data;
2621         u32 tmp, fw_ucode_size, fw_data_size;
2622         u32 i, usec_timeout = 50000; /* Wait for 50 ms */
2623         u32 *fw_ucode_ptr, *fw_data_ptr;
2624         int r;
2625
2626         if (!adev->gfx.mec_fw)
2627                 return -EINVAL;
2628
2629         gfx_v12_0_cp_compute_enable(adev, false);
2630
2631         mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
2632         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
2633
2634         fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
2635                                 le32_to_cpu(mec_hdr->ucode_offset_bytes));
2636         fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
2637
2638         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
2639                                 le32_to_cpu(mec_hdr->data_offset_bytes));
2640         fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
2641
2642         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2643                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2644                                       &adev->gfx.mec.mec_fw_obj,
2645                                       &adev->gfx.mec.mec_fw_gpu_addr,
2646                                       (void **)&fw_ucode_ptr);
2647         if (r) {
2648                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
2649                 gfx_v12_0_mec_fini(adev);
2650                 return r;
2651         }
2652
2653         r = amdgpu_bo_create_reserved(adev,
2654                                       ALIGN(fw_data_size, 64 * 1024) *
2655                                       adev->gfx.mec.num_pipe_per_mec,
2656                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2657                                       &adev->gfx.mec.mec_fw_data_obj,
2658                                       &adev->gfx.mec.mec_fw_data_gpu_addr,
2659                                       (void **)&fw_data_ptr);
2660         if (r) {
2661                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
2662                 gfx_v12_0_mec_fini(adev);
2663                 return r;
2664         }
2665
2666         memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
2667         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2668                 memcpy(fw_data_ptr + i * ALIGN(fw_data_size, 64 * 1024) / 4, fw_data, fw_data_size);
2669         }
2670
2671         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
2672         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
2673         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
2674         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
2675
2676         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2677         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2678         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2679         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2680         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2681
2682         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2683         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2684         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2685         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2686
2687         mutex_lock(&adev->srbm_mutex);
2688         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2689                 soc24_grbm_select(adev, 1, i, 0, 0);
2690
2691                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO,
2692                              lower_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
2693                                            i * ALIGN(fw_data_size, 64 * 1024)));
2694                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2695                              upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
2696                                            i * ALIGN(fw_data_size, 64 * 1024)));
2697
2698                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2699                              lower_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
2700                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2701                              upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
2702         }
2703         mutex_unlock(&adev->srbm_mutex);
2704         soc24_grbm_select(adev, 0, 0, 0, 0);
2705
2706         /* Trigger an invalidation of the L1 instruction caches */
2707         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2708         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2709         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2710
2711         /* Wait for invalidation complete */
2712         for (i = 0; i < usec_timeout; i++) {
2713                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2714                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2715                                        INVALIDATE_DCACHE_COMPLETE))
2716                         break;
2717                 udelay(1);
2718         }
2719
2720         if (i >= usec_timeout) {
2721                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2722                 return -EINVAL;
2723         }
2724
2725         /* Trigger an invalidation of the L1 instruction caches */
2726         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2727         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2728         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2729
2730         /* Wait for invalidation complete */
2731         for (i = 0; i < usec_timeout; i++) {
2732                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2733                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2734                                        INVALIDATE_CACHE_COMPLETE))
2735                         break;
2736                 udelay(1);
2737         }
2738
2739         if (i >= usec_timeout) {
2740                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2741                 return -EINVAL;
2742         }
2743
2744         gfx_v12_0_set_mec_ucode_start_addr(adev);
2745
2746         return 0;
2747 }
2748
2749 static void gfx_v12_0_kiq_setting(struct amdgpu_ring *ring)
2750 {
2751         uint32_t tmp;
2752         struct amdgpu_device *adev = ring->adev;
2753
2754         /* tell RLC which is KIQ queue */
2755         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
2756         tmp &= 0xffffff00;
2757         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
2758         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
2759         tmp |= 0x80;
2760         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
2761 }
2762
2763 static void gfx_v12_0_cp_set_doorbell_range(struct amdgpu_device *adev)
2764 {
2765         /* set graphics engine doorbell range */
2766         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
2767                      (adev->doorbell_index.gfx_ring0 * 2) << 2);
2768         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
2769                      (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
2770
2771         /* set compute engine doorbell range */
2772         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
2773                      (adev->doorbell_index.kiq * 2) << 2);
2774         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
2775                      (adev->doorbell_index.userqueue_end * 2) << 2);
2776 }
2777
2778 static int gfx_v12_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
2779                                   struct amdgpu_mqd_prop *prop)
2780 {
2781         struct v12_gfx_mqd *mqd = m;
2782         uint64_t hqd_gpu_addr, wb_gpu_addr;
2783         uint32_t tmp;
2784         uint32_t rb_bufsz;
2785
2786         /* set up gfx hqd wptr */
2787         mqd->cp_gfx_hqd_wptr = 0;
2788         mqd->cp_gfx_hqd_wptr_hi = 0;
2789
2790         /* set the pointer to the MQD */
2791         mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
2792         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
2793
2794         /* set up mqd control */
2795         tmp = RREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL);
2796         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
2797         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
2798         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
2799         mqd->cp_gfx_mqd_control = tmp;
2800
2801         /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
2802         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID);
2803         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
2804         mqd->cp_gfx_hqd_vmid = 0;
2805
2806         /* set up default queue priority level
2807          * 0x0 = low priority, 0x1 = high priority */
2808         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY);
2809         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, 0);
2810         mqd->cp_gfx_hqd_queue_priority = tmp;
2811
2812         /* set up time quantum */
2813         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM);
2814         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
2815         mqd->cp_gfx_hqd_quantum = tmp;
2816
2817         /* set up gfx hqd base. this is similar as CP_RB_BASE */
2818         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
2819         mqd->cp_gfx_hqd_base = hqd_gpu_addr;
2820         mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
2821
2822         /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
2823         wb_gpu_addr = prop->rptr_gpu_addr;
2824         mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
2825         mqd->cp_gfx_hqd_rptr_addr_hi =
2826                 upper_32_bits(wb_gpu_addr) & 0xffff;
2827
2828         /* set up rb_wptr_poll addr */
2829         wb_gpu_addr = prop->wptr_gpu_addr;
2830         mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
2831         mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
2832
2833         /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
2834         rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
2835         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL);
2836         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
2837         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
2838 #ifdef __BIG_ENDIAN
2839         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
2840 #endif
2841         mqd->cp_gfx_hqd_cntl = tmp;
2842
2843         /* set up cp_doorbell_control */
2844         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
2845         if (prop->use_doorbell) {
2846                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2847                                     DOORBELL_OFFSET, prop->doorbell_index);
2848                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2849                                     DOORBELL_EN, 1);
2850         } else
2851                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2852                                     DOORBELL_EN, 0);
2853         mqd->cp_rb_doorbell_control = tmp;
2854
2855         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2856         mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR);
2857
2858         /* active the queue */
2859         mqd->cp_gfx_hqd_active = 1;
2860
2861         return 0;
2862 }
2863
2864 static int gfx_v12_0_gfx_init_queue(struct amdgpu_ring *ring)
2865 {
2866         struct amdgpu_device *adev = ring->adev;
2867         struct v12_gfx_mqd *mqd = ring->mqd_ptr;
2868         int mqd_idx = ring - &adev->gfx.gfx_ring[0];
2869
2870         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
2871                 memset((void *)mqd, 0, sizeof(*mqd));
2872                 mutex_lock(&adev->srbm_mutex);
2873                 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
2874                 amdgpu_ring_init_mqd(ring);
2875                 soc24_grbm_select(adev, 0, 0, 0, 0);
2876                 mutex_unlock(&adev->srbm_mutex);
2877                 if (adev->gfx.me.mqd_backup[mqd_idx])
2878                         memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
2879         } else {
2880                 /* restore mqd with the backup copy */
2881                 if (adev->gfx.me.mqd_backup[mqd_idx])
2882                         memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
2883                 /* reset the ring */
2884                 ring->wptr = 0;
2885                 *ring->wptr_cpu_addr = 0;
2886                 amdgpu_ring_clear_ring(ring);
2887         }
2888
2889         return 0;
2890 }
2891
2892 static int gfx_v12_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
2893 {
2894         int r, i;
2895         struct amdgpu_ring *ring;
2896
2897         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2898                 ring = &adev->gfx.gfx_ring[i];
2899
2900                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
2901                 if (unlikely(r != 0))
2902                         goto done;
2903
2904                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
2905                 if (!r) {
2906                         r = gfx_v12_0_gfx_init_queue(ring);
2907                         amdgpu_bo_kunmap(ring->mqd_obj);
2908                         ring->mqd_ptr = NULL;
2909                 }
2910                 amdgpu_bo_unreserve(ring->mqd_obj);
2911                 if (r)
2912                         goto done;
2913         }
2914
2915         r = amdgpu_gfx_enable_kgq(adev, 0);
2916         if (r)
2917                 goto done;
2918
2919         r = gfx_v12_0_cp_gfx_start(adev);
2920         if (r)
2921                 goto done;
2922
2923         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2924                 ring = &adev->gfx.gfx_ring[i];
2925                 ring->sched.ready = true;
2926         }
2927 done:
2928         return r;
2929 }
2930
2931 static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
2932                                       struct amdgpu_mqd_prop *prop)
2933 {
2934         struct v12_compute_mqd *mqd = m;
2935         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
2936         uint32_t tmp;
2937
2938         mqd->header = 0xC0310800;
2939         mqd->compute_pipelinestat_enable = 0x00000001;
2940         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
2941         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
2942         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
2943         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
2944         mqd->compute_misc_reserved = 0x00000007;
2945
2946         eop_base_addr = prop->eop_gpu_addr >> 8;
2947         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
2948         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
2949
2950         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2951         tmp = RREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL);
2952         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
2953                         (order_base_2(GFX12_MEC_HPD_SIZE / 4) - 1));
2954
2955         mqd->cp_hqd_eop_control = tmp;
2956
2957         /* enable doorbell? */
2958         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
2959
2960         if (prop->use_doorbell) {
2961                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2962                                     DOORBELL_OFFSET, prop->doorbell_index);
2963                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2964                                     DOORBELL_EN, 1);
2965                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2966                                     DOORBELL_SOURCE, 0);
2967                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2968                                     DOORBELL_HIT, 0);
2969         } else {
2970                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2971                                     DOORBELL_EN, 0);
2972         }
2973
2974         mqd->cp_hqd_pq_doorbell_control = tmp;
2975
2976         /* disable the queue if it's active */
2977         mqd->cp_hqd_dequeue_request = 0;
2978         mqd->cp_hqd_pq_rptr = 0;
2979         mqd->cp_hqd_pq_wptr_lo = 0;
2980         mqd->cp_hqd_pq_wptr_hi = 0;
2981
2982         /* set the pointer to the MQD */
2983         mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
2984         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
2985
2986         /* set MQD vmid to 0 */
2987         tmp = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
2988         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
2989         mqd->cp_mqd_control = tmp;
2990
2991         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2992         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
2993         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
2994         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
2995
2996         /* set up the HQD, this is similar to CP_RB0_CNTL */
2997         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL);
2998         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
2999                             (order_base_2(prop->queue_size / 4) - 1));
3000         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3001                             (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3002         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3003         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
3004         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3005         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3006         mqd->cp_hqd_pq_control = tmp;
3007
3008         /* set the wb address whether it's enabled or not */
3009         wb_gpu_addr = prop->rptr_gpu_addr;
3010         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3011         mqd->cp_hqd_pq_rptr_report_addr_hi =
3012                 upper_32_bits(wb_gpu_addr) & 0xffff;
3013
3014         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3015         wb_gpu_addr = prop->wptr_gpu_addr;
3016         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3017         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3018
3019         tmp = 0;
3020         /* enable the doorbell if requested */
3021         if (prop->use_doorbell) {
3022                 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
3023                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3024                                 DOORBELL_OFFSET, prop->doorbell_index);
3025
3026                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3027                                     DOORBELL_EN, 1);
3028                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3029                                     DOORBELL_SOURCE, 0);
3030                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3031                                     DOORBELL_HIT, 0);
3032         }
3033
3034         mqd->cp_hqd_pq_doorbell_control = tmp;
3035
3036         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3037         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR);
3038
3039         /* set the vmid for the queue */
3040         mqd->cp_hqd_vmid = 0;
3041
3042         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE);
3043         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
3044         mqd->cp_hqd_persistent_state = tmp;
3045
3046         /* set MIN_IB_AVAIL_SIZE */
3047         tmp = RREG32_SOC15(GC, 0, regCP_HQD_IB_CONTROL);
3048         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3049         mqd->cp_hqd_ib_control = tmp;
3050
3051         /* set static priority for a compute queue/ring */
3052         mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
3053         mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
3054
3055         mqd->cp_hqd_active = prop->hqd_active;
3056
3057         return 0;
3058 }
3059
3060 static int gfx_v12_0_kiq_init_register(struct amdgpu_ring *ring)
3061 {
3062         struct amdgpu_device *adev = ring->adev;
3063         struct v12_compute_mqd *mqd = ring->mqd_ptr;
3064         int j;
3065
3066         /* inactivate the queue */
3067         if (amdgpu_sriov_vf(adev))
3068                 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
3069
3070         /* disable wptr polling */
3071         WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3072
3073         /* write the EOP addr */
3074         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
3075                mqd->cp_hqd_eop_base_addr_lo);
3076         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
3077                mqd->cp_hqd_eop_base_addr_hi);
3078
3079         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3080         WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
3081                mqd->cp_hqd_eop_control);
3082
3083         /* enable doorbell? */
3084         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
3085                mqd->cp_hqd_pq_doorbell_control);
3086
3087         /* disable the queue if it's active */
3088         if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
3089                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
3090                 for (j = 0; j < adev->usec_timeout; j++) {
3091                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
3092                                 break;
3093                         udelay(1);
3094                 }
3095                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
3096                        mqd->cp_hqd_dequeue_request);
3097                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
3098                        mqd->cp_hqd_pq_rptr);
3099                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
3100                        mqd->cp_hqd_pq_wptr_lo);
3101                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
3102                        mqd->cp_hqd_pq_wptr_hi);
3103         }
3104
3105         /* set the pointer to the MQD */
3106         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
3107                mqd->cp_mqd_base_addr_lo);
3108         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
3109                mqd->cp_mqd_base_addr_hi);
3110
3111         /* set MQD vmid to 0 */
3112         WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
3113                mqd->cp_mqd_control);
3114
3115         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3116         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
3117                mqd->cp_hqd_pq_base_lo);
3118         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
3119                mqd->cp_hqd_pq_base_hi);
3120
3121         /* set up the HQD, this is similar to CP_RB0_CNTL */
3122         WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
3123                mqd->cp_hqd_pq_control);
3124
3125         /* set the wb address whether it's enabled or not */
3126         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
3127                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3128         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3129                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3130
3131         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3132         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
3133                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3134         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3135                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3136
3137         /* enable the doorbell if requested */
3138         if (ring->use_doorbell) {
3139                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
3140                         (adev->doorbell_index.kiq * 2) << 2);
3141                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
3142                         (adev->doorbell_index.userqueue_end * 2) << 2);
3143         }
3144
3145         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
3146                mqd->cp_hqd_pq_doorbell_control);
3147
3148         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3149         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
3150                mqd->cp_hqd_pq_wptr_lo);
3151         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
3152                mqd->cp_hqd_pq_wptr_hi);
3153
3154         /* set the vmid for the queue */
3155         WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
3156
3157         WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
3158                mqd->cp_hqd_persistent_state);
3159
3160         /* activate the queue */
3161         WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
3162                mqd->cp_hqd_active);
3163
3164         if (ring->use_doorbell)
3165                 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3166
3167         return 0;
3168 }
3169
3170 static int gfx_v12_0_kiq_init_queue(struct amdgpu_ring *ring)
3171 {
3172         struct amdgpu_device *adev = ring->adev;
3173         struct v12_compute_mqd *mqd = ring->mqd_ptr;
3174         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3175
3176         gfx_v12_0_kiq_setting(ring);
3177
3178         if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
3179                 /* reset MQD to a clean status */
3180                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3181                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
3182
3183                 /* reset ring buffer */
3184                 ring->wptr = 0;
3185                 amdgpu_ring_clear_ring(ring);
3186
3187                 mutex_lock(&adev->srbm_mutex);
3188                 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3189                 gfx_v12_0_kiq_init_register(ring);
3190                 soc24_grbm_select(adev, 0, 0, 0, 0);
3191                 mutex_unlock(&adev->srbm_mutex);
3192         } else {
3193                 memset((void *)mqd, 0, sizeof(*mqd));
3194                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3195                         amdgpu_ring_clear_ring(ring);
3196                 mutex_lock(&adev->srbm_mutex);
3197                 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3198                 amdgpu_ring_init_mqd(ring);
3199                 gfx_v12_0_kiq_init_register(ring);
3200                 soc24_grbm_select(adev, 0, 0, 0, 0);
3201                 mutex_unlock(&adev->srbm_mutex);
3202
3203                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3204                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3205         }
3206
3207         return 0;
3208 }
3209
3210 static int gfx_v12_0_kcq_init_queue(struct amdgpu_ring *ring)
3211 {
3212         struct amdgpu_device *adev = ring->adev;
3213         struct v12_compute_mqd *mqd = ring->mqd_ptr;
3214         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3215
3216         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3217                 memset((void *)mqd, 0, sizeof(*mqd));
3218                 mutex_lock(&adev->srbm_mutex);
3219                 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3220                 amdgpu_ring_init_mqd(ring);
3221                 soc24_grbm_select(adev, 0, 0, 0, 0);
3222                 mutex_unlock(&adev->srbm_mutex);
3223
3224                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3225                         memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3226         } else {
3227                 /* restore MQD to a clean status */
3228                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3229                         memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
3230                 /* reset ring buffer */
3231                 ring->wptr = 0;
3232                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
3233                 amdgpu_ring_clear_ring(ring);
3234         }
3235
3236         return 0;
3237 }
3238
3239 static int gfx_v12_0_kiq_resume(struct amdgpu_device *adev)
3240 {
3241         struct amdgpu_ring *ring;
3242         int r;
3243
3244         ring = &adev->gfx.kiq[0].ring;
3245
3246         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3247         if (unlikely(r != 0))
3248                 return r;
3249
3250         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3251         if (unlikely(r != 0)) {
3252                 amdgpu_bo_unreserve(ring->mqd_obj);
3253                 return r;
3254         }
3255
3256         gfx_v12_0_kiq_init_queue(ring);
3257         amdgpu_bo_kunmap(ring->mqd_obj);
3258         ring->mqd_ptr = NULL;
3259         amdgpu_bo_unreserve(ring->mqd_obj);
3260         ring->sched.ready = true;
3261         return 0;
3262 }
3263
3264 static int gfx_v12_0_kcq_resume(struct amdgpu_device *adev)
3265 {
3266         struct amdgpu_ring *ring = NULL;
3267         int r = 0, i;
3268
3269         if (!amdgpu_async_gfx_ring)
3270                 gfx_v12_0_cp_compute_enable(adev, true);
3271
3272         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3273                 ring = &adev->gfx.compute_ring[i];
3274
3275                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3276                 if (unlikely(r != 0))
3277                         goto done;
3278                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3279                 if (!r) {
3280                         r = gfx_v12_0_kcq_init_queue(ring);
3281                         amdgpu_bo_kunmap(ring->mqd_obj);
3282                         ring->mqd_ptr = NULL;
3283                 }
3284                 amdgpu_bo_unreserve(ring->mqd_obj);
3285                 if (r)
3286                         goto done;
3287         }
3288
3289         r = amdgpu_gfx_enable_kcq(adev, 0);
3290 done:
3291         return r;
3292 }
3293
3294 static int gfx_v12_0_cp_resume(struct amdgpu_device *adev)
3295 {
3296         int r, i;
3297         struct amdgpu_ring *ring;
3298
3299         if (!(adev->flags & AMD_IS_APU))
3300                 gfx_v12_0_enable_gui_idle_interrupt(adev, false);
3301
3302         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
3303                 /* legacy firmware loading */
3304                 r = gfx_v12_0_cp_gfx_load_microcode(adev);
3305                 if (r)
3306                         return r;
3307
3308                 r = gfx_v12_0_cp_compute_load_microcode_rs64(adev);
3309                 if (r)
3310                         return r;
3311         }
3312
3313         gfx_v12_0_cp_set_doorbell_range(adev);
3314
3315         if (amdgpu_async_gfx_ring) {
3316                 gfx_v12_0_cp_compute_enable(adev, true);
3317                 gfx_v12_0_cp_gfx_enable(adev, true);
3318         }
3319
3320         if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
3321                 r = amdgpu_mes_kiq_hw_init(adev);
3322         else
3323                 r = gfx_v12_0_kiq_resume(adev);
3324         if (r)
3325                 return r;
3326
3327         r = gfx_v12_0_kcq_resume(adev);
3328         if (r)
3329                 return r;
3330
3331         if (!amdgpu_async_gfx_ring) {
3332                 r = gfx_v12_0_cp_gfx_resume(adev);
3333                 if (r)
3334                         return r;
3335         } else {
3336                 r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
3337                 if (r)
3338                         return r;
3339         }
3340
3341         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3342                 ring = &adev->gfx.gfx_ring[i];
3343                 r = amdgpu_ring_test_helper(ring);
3344                 if (r)
3345                         return r;
3346         }
3347
3348         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3349                 ring = &adev->gfx.compute_ring[i];
3350                 r = amdgpu_ring_test_helper(ring);
3351                 if (r)
3352                         return r;
3353         }
3354
3355         return 0;
3356 }
3357
3358 static void gfx_v12_0_cp_enable(struct amdgpu_device *adev, bool enable)
3359 {
3360         gfx_v12_0_cp_gfx_enable(adev, enable);
3361         gfx_v12_0_cp_compute_enable(adev, enable);
3362 }
3363
3364 static int gfx_v12_0_gfxhub_enable(struct amdgpu_device *adev)
3365 {
3366         int r;
3367         bool value;
3368
3369         r = adev->gfxhub.funcs->gart_enable(adev);
3370         if (r)
3371                 return r;
3372
3373         adev->hdp.funcs->flush_hdp(adev, NULL);
3374
3375         value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
3376                 false : true;
3377
3378         adev->gfxhub.funcs->set_fault_enable_default(adev, value);
3379         /* TODO investigate why this and the hdp flush above is needed,
3380          * are we missing a flush somewhere else? */
3381         adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
3382
3383         return 0;
3384 }
3385
3386 static int get_gb_addr_config(struct amdgpu_device *adev)
3387 {
3388         u32 gb_addr_config;
3389
3390         gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
3391         if (gb_addr_config == 0)
3392                 return -EINVAL;
3393
3394         adev->gfx.config.gb_addr_config_fields.num_pkrs =
3395                 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
3396
3397         adev->gfx.config.gb_addr_config = gb_addr_config;
3398
3399         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
3400                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3401                                       GB_ADDR_CONFIG, NUM_PIPES);
3402
3403         adev->gfx.config.max_tile_pipes =
3404                 adev->gfx.config.gb_addr_config_fields.num_pipes;
3405
3406         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
3407                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3408                                       GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
3409         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
3410                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3411                                       GB_ADDR_CONFIG, NUM_RB_PER_SE);
3412         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
3413                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3414                                       GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
3415         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
3416                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3417                                       GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
3418
3419         return 0;
3420 }
3421
3422 static void gfx_v12_0_disable_gpa_mode(struct amdgpu_device *adev)
3423 {
3424         uint32_t data;
3425
3426         data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
3427         data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
3428         WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
3429
3430         data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
3431         data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
3432         WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
3433 }
3434
3435 static int gfx_v12_0_hw_init(void *handle)
3436 {
3437         int r;
3438         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3439
3440         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
3441                 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
3442                         /* RLC autoload sequence 1: Program rlc ram */
3443                         if (adev->gfx.imu.funcs->program_rlc_ram)
3444                                 adev->gfx.imu.funcs->program_rlc_ram(adev);
3445                 }
3446                 /* rlc autoload firmware */
3447                 r = gfx_v12_0_rlc_backdoor_autoload_enable(adev);
3448                 if (r)
3449                         return r;
3450         } else {
3451                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
3452                         if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
3453                                 if (adev->gfx.imu.funcs->load_microcode)
3454                                         adev->gfx.imu.funcs->load_microcode(adev);
3455                                 if (adev->gfx.imu.funcs->setup_imu)
3456                                         adev->gfx.imu.funcs->setup_imu(adev);
3457                                 if (adev->gfx.imu.funcs->start_imu)
3458                                         adev->gfx.imu.funcs->start_imu(adev);
3459                         }
3460
3461                         /* disable gpa mode in backdoor loading */
3462                         gfx_v12_0_disable_gpa_mode(adev);
3463                 }
3464         }
3465
3466         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
3467             (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
3468                 r = gfx_v12_0_wait_for_rlc_autoload_complete(adev);
3469                 if (r) {
3470                         dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
3471                         return r;
3472                 }
3473         }
3474
3475         adev->gfx.is_poweron = true;
3476
3477         if (get_gb_addr_config(adev))
3478                 DRM_WARN("Invalid gb_addr_config !\n");
3479
3480         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
3481                 gfx_v12_0_config_gfx_rs64(adev);
3482
3483         r = gfx_v12_0_gfxhub_enable(adev);
3484         if (r)
3485                 return r;
3486
3487         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT ||
3488              adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) &&
3489              (amdgpu_dpm == 1)) {
3490                 /**
3491                  * For gfx 12, rlc firmware loading relies on smu firmware is
3492                  * loaded firstly, so in direct type, it has to load smc ucode
3493                  * here before rlc.
3494                  */
3495                 r = amdgpu_pm_load_smu_firmware(adev, NULL);
3496                 if (r)
3497                         return r;
3498         }
3499
3500         gfx_v12_0_constants_init(adev);
3501
3502         if (adev->nbio.funcs->gc_doorbell_init)
3503                 adev->nbio.funcs->gc_doorbell_init(adev);
3504
3505         r = gfx_v12_0_rlc_resume(adev);
3506         if (r)
3507                 return r;
3508
3509         /*
3510          * init golden registers and rlc resume may override some registers,
3511          * reconfig them here
3512          */
3513         gfx_v12_0_tcp_harvest(adev);
3514
3515         r = gfx_v12_0_cp_resume(adev);
3516         if (r)
3517                 return r;
3518
3519         return r;
3520 }
3521
3522 static int gfx_v12_0_kiq_disable_kgq(struct amdgpu_device *adev)
3523 {
3524         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
3525         struct amdgpu_ring *kiq_ring = &kiq->ring;
3526         int i, r = 0;
3527
3528         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
3529                 return -EINVAL;
3530
3531         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
3532                                         adev->gfx.num_gfx_rings))
3533                 return -ENOMEM;
3534
3535         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
3536                 kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i],
3537                                            PREEMPT_QUEUES, 0, 0);
3538
3539         if (adev->gfx.kiq[0].ring.sched.ready)
3540                 r = amdgpu_ring_test_helper(kiq_ring);
3541
3542         return r;
3543 }
3544
3545 static int gfx_v12_0_hw_fini(void *handle)
3546 {
3547         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3548         int r;
3549         uint32_t tmp;
3550
3551         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3552         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3553
3554         if (!adev->no_hw_access) {
3555                 if (amdgpu_async_gfx_ring) {
3556                         r = gfx_v12_0_kiq_disable_kgq(adev);
3557                         if (r)
3558                                 DRM_ERROR("KGQ disable failed\n");
3559                 }
3560
3561                 if (amdgpu_gfx_disable_kcq(adev, 0))
3562                         DRM_ERROR("KCQ disable failed\n");
3563
3564                 amdgpu_mes_kiq_hw_fini(adev);
3565         }
3566
3567         if (amdgpu_sriov_vf(adev)) {
3568                 gfx_v12_0_cp_gfx_enable(adev, false);
3569                 /* Program KIQ position of RLC_CP_SCHEDULERS during destroy */
3570                 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
3571                 tmp &= 0xffffff00;
3572                 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3573
3574                 return 0;
3575         }
3576         gfx_v12_0_cp_enable(adev, false);
3577         gfx_v12_0_enable_gui_idle_interrupt(adev, false);
3578
3579         adev->gfxhub.funcs->gart_disable(adev);
3580
3581         adev->gfx.is_poweron = false;
3582
3583         return 0;
3584 }
3585
3586 static int gfx_v12_0_suspend(void *handle)
3587 {
3588         return gfx_v12_0_hw_fini(handle);
3589 }
3590
3591 static int gfx_v12_0_resume(void *handle)
3592 {
3593         return gfx_v12_0_hw_init(handle);
3594 }
3595
3596 static bool gfx_v12_0_is_idle(void *handle)
3597 {
3598         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3599
3600         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
3601                                 GRBM_STATUS, GUI_ACTIVE))
3602                 return false;
3603         else
3604                 return true;
3605 }
3606
3607 static int gfx_v12_0_wait_for_idle(void *handle)
3608 {
3609         unsigned i;
3610         u32 tmp;
3611         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3612
3613         for (i = 0; i < adev->usec_timeout; i++) {
3614                 /* read MC_STATUS */
3615                 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
3616                         GRBM_STATUS__GUI_ACTIVE_MASK;
3617
3618                 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
3619                         return 0;
3620                 udelay(1);
3621         }
3622         return -ETIMEDOUT;
3623 }
3624
3625 static uint64_t gfx_v12_0_get_gpu_clock_counter(struct amdgpu_device *adev)
3626 {
3627         uint64_t clock = 0;
3628
3629         if (adev->smuio.funcs &&
3630             adev->smuio.funcs->get_gpu_clock_counter)
3631                 clock = adev->smuio.funcs->get_gpu_clock_counter(adev);
3632         else
3633                 dev_warn(adev->dev, "query gpu clock counter is not supported\n");
3634
3635         return clock;
3636 }
3637
3638 static int gfx_v12_0_early_init(void *handle)
3639 {
3640         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3641
3642         adev->gfx.funcs = &gfx_v12_0_gfx_funcs;
3643
3644         adev->gfx.num_gfx_rings = GFX12_NUM_GFX_RINGS;
3645         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
3646                                           AMDGPU_MAX_COMPUTE_RINGS);
3647
3648         gfx_v12_0_set_kiq_pm4_funcs(adev);
3649         gfx_v12_0_set_ring_funcs(adev);
3650         gfx_v12_0_set_irq_funcs(adev);
3651         gfx_v12_0_set_rlc_funcs(adev);
3652         gfx_v12_0_set_mqd_funcs(adev);
3653         gfx_v12_0_set_imu_funcs(adev);
3654
3655         gfx_v12_0_init_rlcg_reg_access_ctrl(adev);
3656
3657         return gfx_v12_0_init_microcode(adev);
3658 }
3659
3660 static int gfx_v12_0_late_init(void *handle)
3661 {
3662         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3663         int r;
3664
3665         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
3666         if (r)
3667                 return r;
3668
3669         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
3670         if (r)
3671                 return r;
3672
3673         return 0;
3674 }
3675
3676 static bool gfx_v12_0_is_rlc_enabled(struct amdgpu_device *adev)
3677 {
3678         uint32_t rlc_cntl;
3679
3680         /* if RLC is not enabled, do nothing */
3681         rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
3682         return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
3683 }
3684
3685 static void gfx_v12_0_set_safe_mode(struct amdgpu_device *adev,
3686                                     int xcc_id)
3687 {
3688         uint32_t data;
3689         unsigned i;
3690
3691         data = RLC_SAFE_MODE__CMD_MASK;
3692         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
3693
3694         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
3695
3696         /* wait for RLC_SAFE_MODE */
3697         for (i = 0; i < adev->usec_timeout; i++) {
3698                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
3699                                    RLC_SAFE_MODE, CMD))
3700                         break;
3701                 udelay(1);
3702         }
3703 }
3704
3705 static void gfx_v12_0_unset_safe_mode(struct amdgpu_device *adev,
3706                                       int xcc_id)
3707 {
3708         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
3709 }
3710
3711 static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev,
3712                                       bool enable)
3713 {
3714         uint32_t def, data;
3715
3716         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
3717                 return;
3718
3719         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3720
3721         if (enable)
3722                 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
3723         else
3724                 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
3725
3726         if (def != data)
3727                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3728 }
3729
3730 static void gfx_v12_0_update_spm_vmid(struct amdgpu_device *adev,
3731                                       struct amdgpu_ring *ring,
3732                                       unsigned vmid)
3733 {
3734         u32 reg, data;
3735
3736         reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
3737         if (amdgpu_sriov_is_pp_one_vf(adev))
3738                 data = RREG32_NO_KIQ(reg);
3739         else
3740                 data = RREG32(reg);
3741
3742         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
3743         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
3744
3745         if (amdgpu_sriov_is_pp_one_vf(adev))
3746                 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
3747         else
3748                 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
3749
3750         if (ring
3751             && amdgpu_sriov_is_pp_one_vf(adev)
3752             && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
3753                 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
3754                 uint32_t reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
3755                 amdgpu_ring_emit_wreg(ring, reg, data);
3756         }
3757 }
3758
3759 static const struct amdgpu_rlc_funcs gfx_v12_0_rlc_funcs = {
3760         .is_rlc_enabled = gfx_v12_0_is_rlc_enabled,
3761         .set_safe_mode = gfx_v12_0_set_safe_mode,
3762         .unset_safe_mode = gfx_v12_0_unset_safe_mode,
3763         .init = gfx_v12_0_rlc_init,
3764         .get_csb_size = gfx_v12_0_get_csb_size,
3765         .get_csb_buffer = gfx_v12_0_get_csb_buffer,
3766         .resume = gfx_v12_0_rlc_resume,
3767         .stop = gfx_v12_0_rlc_stop,
3768         .reset = gfx_v12_0_rlc_reset,
3769         .start = gfx_v12_0_rlc_start,
3770         .update_spm_vmid = gfx_v12_0_update_spm_vmid,
3771 };
3772
3773 #if 0
3774 static void gfx_v12_cntl_power_gating(struct amdgpu_device *adev, bool enable)
3775 {
3776         /* TODO */
3777 }
3778
3779 static void gfx_v12_cntl_pg(struct amdgpu_device *adev, bool enable)
3780 {
3781         /* TODO */
3782 }
3783 #endif
3784
3785 static int gfx_v12_0_set_powergating_state(void *handle,
3786                                            enum amd_powergating_state state)
3787 {
3788         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3789         bool enable = (state == AMD_PG_STATE_GATE);
3790
3791         if (amdgpu_sriov_vf(adev))
3792                 return 0;
3793
3794         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3795         case IP_VERSION(12, 0, 0):
3796         case IP_VERSION(12, 0, 1):
3797                 amdgpu_gfx_off_ctrl(adev, enable);
3798                 break;
3799         default:
3800                 break;
3801         }
3802
3803         return 0;
3804 }
3805
3806 static void gfx_v12_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
3807                                                        bool enable)
3808 {
3809         uint32_t def, data;
3810
3811         if (!(adev->cg_flags &
3812               (AMD_CG_SUPPORT_GFX_CGCG |
3813               AMD_CG_SUPPORT_GFX_CGLS |
3814               AMD_CG_SUPPORT_GFX_3D_CGCG |
3815               AMD_CG_SUPPORT_GFX_3D_CGLS)))
3816                 return;
3817
3818         if (enable) {
3819                 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3820
3821                 /* unset CGCG override */
3822                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
3823                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
3824                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
3825                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
3826                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
3827                     adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
3828                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
3829
3830                 /* update CGCG override bits */
3831                 if (def != data)
3832                         WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3833
3834                 /* enable cgcg FSM(0x0000363F) */
3835                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
3836
3837                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
3838                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
3839                         data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
3840                                  RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
3841                 }
3842
3843                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
3844                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
3845                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
3846                                  RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
3847                 }
3848
3849                 if (def != data)
3850                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
3851
3852                 /* Program RLC_CGCG_CGLS_CTRL_3D */
3853                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
3854
3855                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
3856                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
3857                         data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
3858                                  RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
3859                 }
3860
3861                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
3862                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
3863                         data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
3864                                  RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
3865                 }
3866
3867                 if (def != data)
3868                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
3869
3870                 /* set IDLE_POLL_COUNT(0x00900100) */
3871                 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
3872
3873                 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
3874                 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
3875                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
3876
3877                 if (def != data)
3878                         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
3879
3880                 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
3881                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
3882                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
3883                 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
3884                 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
3885                 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
3886
3887                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
3888                 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
3889                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
3890
3891                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
3892                 if (adev->sdma.num_instances > 1) {
3893                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
3894                         data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
3895                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
3896                 }
3897         } else {
3898                 /* Program RLC_CGCG_CGLS_CTRL */
3899                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
3900
3901                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
3902                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
3903
3904                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
3905                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
3906
3907                 if (def != data)
3908                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
3909
3910                 /* Program RLC_CGCG_CGLS_CTRL_3D */
3911                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
3912
3913                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
3914                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
3915                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
3916                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
3917
3918                 if (def != data)
3919                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
3920
3921                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
3922                 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
3923                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
3924
3925                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
3926                 if (adev->sdma.num_instances > 1) {
3927                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
3928                         data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
3929                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
3930                 }
3931         }
3932 }
3933
3934 static void gfx_v12_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
3935                                                        bool enable)
3936 {
3937         uint32_t data, def;
3938         if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
3939                 return;
3940
3941         /* It is disabled by HW by default */
3942         if (enable) {
3943                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
3944                         /* 1 - RLC_CGTT_MGCG_OVERRIDE */
3945                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3946
3947                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
3948                                   RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
3949                                   RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
3950
3951                         if (def != data)
3952                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3953                 }
3954         } else {
3955                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
3956                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3957
3958                         data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
3959                                  RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
3960                                  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
3961
3962                         if (def != data)
3963                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3964                 }
3965         }
3966 }
3967
3968 static void gfx_v12_0_update_repeater_fgcg(struct amdgpu_device *adev,
3969                                            bool enable)
3970 {
3971         uint32_t def, data;
3972
3973         if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
3974                 return;
3975
3976         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3977
3978         if (enable)
3979                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK |
3980                                   RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK);
3981         else
3982                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK |
3983                                 RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK;
3984
3985         if (def != data)
3986                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
3987 }
3988
3989 static void gfx_v12_0_update_sram_fgcg(struct amdgpu_device *adev,
3990                                        bool enable)
3991 {
3992         uint32_t def, data;
3993
3994         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
3995                 return;
3996
3997         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
3998
3999         if (enable)
4000                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
4001         else
4002                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
4003
4004         if (def != data)
4005                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4006 }
4007
4008 static int gfx_v12_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4009                                             bool enable)
4010 {
4011         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4012
4013         gfx_v12_0_update_coarse_grain_clock_gating(adev, enable);
4014
4015         gfx_v12_0_update_medium_grain_clock_gating(adev, enable);
4016
4017         gfx_v12_0_update_repeater_fgcg(adev, enable);
4018
4019         gfx_v12_0_update_sram_fgcg(adev, enable);
4020
4021         gfx_v12_0_update_perf_clk(adev, enable);
4022
4023         if (adev->cg_flags &
4024             (AMD_CG_SUPPORT_GFX_MGCG |
4025              AMD_CG_SUPPORT_GFX_CGLS |
4026              AMD_CG_SUPPORT_GFX_CGCG |
4027              AMD_CG_SUPPORT_GFX_3D_CGCG |
4028              AMD_CG_SUPPORT_GFX_3D_CGLS))
4029                 gfx_v12_0_enable_gui_idle_interrupt(adev, enable);
4030
4031         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4032
4033         return 0;
4034 }
4035
4036 static int gfx_v12_0_set_clockgating_state(void *handle,
4037                                            enum amd_clockgating_state state)
4038 {
4039         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4040
4041         if (amdgpu_sriov_vf(adev))
4042                 return 0;
4043
4044         switch (adev->ip_versions[GC_HWIP][0]) {
4045         case IP_VERSION(12, 0, 0):
4046         case IP_VERSION(12, 0, 1):
4047                 gfx_v12_0_update_gfx_clock_gating(adev,
4048                                                   state == AMD_CG_STATE_GATE);
4049                 break;
4050         default:
4051                 break;
4052         }
4053
4054         return 0;
4055 }
4056
4057 static void gfx_v12_0_get_clockgating_state(void *handle, u64 *flags)
4058 {
4059         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4060         int data;
4061
4062         /* AMD_CG_SUPPORT_GFX_MGCG */
4063         data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4064         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
4065                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
4066
4067         /* AMD_CG_SUPPORT_REPEATER_FGCG */
4068         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
4069                 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
4070
4071         /* AMD_CG_SUPPORT_GFX_FGCG */
4072         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
4073                 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
4074
4075         /* AMD_CG_SUPPORT_GFX_PERF_CLK */
4076         if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
4077                 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
4078
4079         /* AMD_CG_SUPPORT_GFX_CGCG */
4080         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
4081         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
4082                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
4083
4084         /* AMD_CG_SUPPORT_GFX_CGLS */
4085         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
4086                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
4087
4088         /* AMD_CG_SUPPORT_GFX_3D_CGCG */
4089         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
4090         if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
4091                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
4092
4093         /* AMD_CG_SUPPORT_GFX_3D_CGLS */
4094         if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
4095                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
4096 }
4097
4098 static u64 gfx_v12_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
4099 {
4100         /* gfx12 is 32bit rptr*/
4101         return *(uint32_t *)ring->rptr_cpu_addr;
4102 }
4103
4104 static u64 gfx_v12_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
4105 {
4106         struct amdgpu_device *adev = ring->adev;
4107         u64 wptr;
4108
4109         /* XXX check if swapping is necessary on BE */
4110         if (ring->use_doorbell) {
4111                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
4112         } else {
4113                 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
4114                 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
4115         }
4116
4117         return wptr;
4118 }
4119
4120 static void gfx_v12_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
4121 {
4122         struct amdgpu_device *adev = ring->adev;
4123         uint32_t *wptr_saved;
4124         uint32_t *is_queue_unmap;
4125         uint64_t aggregated_db_index;
4126         uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_GFX].mqd_size;
4127         uint64_t wptr_tmp;
4128
4129         if (ring->is_mes_queue) {
4130                 wptr_saved = (uint32_t *)(ring->mqd_ptr + mqd_size);
4131                 is_queue_unmap = (uint32_t *)(ring->mqd_ptr + mqd_size +
4132                                               sizeof(uint32_t));
4133                 aggregated_db_index =
4134                         amdgpu_mes_get_aggregated_doorbell_index(adev,
4135                                                                  ring->hw_prio);
4136
4137                 wptr_tmp = ring->wptr & ring->buf_mask;
4138                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, wptr_tmp);
4139                 *wptr_saved = wptr_tmp;
4140                 /* assume doorbell always being used by mes mapped queue */
4141                 if (*is_queue_unmap) {
4142                         WDOORBELL64(aggregated_db_index, wptr_tmp);
4143                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
4144                 } else {
4145                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
4146
4147                         if (*is_queue_unmap)
4148                                 WDOORBELL64(aggregated_db_index, wptr_tmp);
4149                 }
4150         } else {
4151                 if (ring->use_doorbell) {
4152                         /* XXX check if swapping is necessary on BE */
4153                         atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
4154                                      ring->wptr);
4155                         WDOORBELL64(ring->doorbell_index, ring->wptr);
4156                 } else {
4157                         WREG32_SOC15(GC, 0, regCP_RB0_WPTR,
4158                                      lower_32_bits(ring->wptr));
4159                         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI,
4160                                      upper_32_bits(ring->wptr));
4161                 }
4162         }
4163 }
4164
4165 static u64 gfx_v12_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
4166 {
4167         /* gfx12 hardware is 32bit rptr */
4168         return *(uint32_t *)ring->rptr_cpu_addr;
4169 }
4170
4171 static u64 gfx_v12_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
4172 {
4173         u64 wptr;
4174
4175         /* XXX check if swapping is necessary on BE */
4176         if (ring->use_doorbell)
4177                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
4178         else
4179                 BUG();
4180         return wptr;
4181 }
4182
4183 static void gfx_v12_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
4184 {
4185         struct amdgpu_device *adev = ring->adev;
4186         uint32_t *wptr_saved;
4187         uint32_t *is_queue_unmap;
4188         uint64_t aggregated_db_index;
4189         uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size;
4190         uint64_t wptr_tmp;
4191
4192         if (ring->is_mes_queue) {
4193                 wptr_saved = (uint32_t *)(ring->mqd_ptr + mqd_size);
4194                 is_queue_unmap = (uint32_t *)(ring->mqd_ptr + mqd_size +
4195                                               sizeof(uint32_t));
4196                 aggregated_db_index =
4197                         amdgpu_mes_get_aggregated_doorbell_index(adev,
4198                                                                  ring->hw_prio);
4199
4200                 wptr_tmp = ring->wptr & ring->buf_mask;
4201                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, wptr_tmp);
4202                 *wptr_saved = wptr_tmp;
4203                 /* assume doorbell always used by mes mapped queue */
4204                 if (*is_queue_unmap) {
4205                         WDOORBELL64(aggregated_db_index, wptr_tmp);
4206                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
4207                 } else {
4208                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
4209
4210                         if (*is_queue_unmap)
4211                                 WDOORBELL64(aggregated_db_index, wptr_tmp);
4212                 }
4213         } else {
4214                 /* XXX check if swapping is necessary on BE */
4215                 if (ring->use_doorbell) {
4216                         atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
4217                                      ring->wptr);
4218                         WDOORBELL64(ring->doorbell_index, ring->wptr);
4219                 } else {
4220                         BUG(); /* only DOORBELL method supported on gfx12 now */
4221                 }
4222         }
4223 }
4224
4225 static void gfx_v12_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
4226 {
4227         struct amdgpu_device *adev = ring->adev;
4228         u32 ref_and_mask, reg_mem_engine;
4229         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
4230
4231         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
4232                 switch (ring->me) {
4233                 case 1:
4234                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
4235                         break;
4236                 case 2:
4237                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
4238                         break;
4239                 default:
4240                         return;
4241                 }
4242                 reg_mem_engine = 0;
4243         } else {
4244                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
4245                 reg_mem_engine = 1; /* pfp */
4246         }
4247
4248         gfx_v12_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
4249                                adev->nbio.funcs->get_hdp_flush_req_offset(adev),
4250                                adev->nbio.funcs->get_hdp_flush_done_offset(adev),
4251                                ref_and_mask, ref_and_mask, 0x20);
4252 }
4253
4254 static void gfx_v12_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
4255                                        struct amdgpu_job *job,
4256                                        struct amdgpu_ib *ib,
4257                                        uint32_t flags)
4258 {
4259         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
4260         u32 header, control = 0;
4261
4262         BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE);
4263
4264         header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4265
4266         control |= ib->length_dw | (vmid << 24);
4267
4268         if (ring->is_mes_queue)
4269                 /* inherit vmid from mqd */
4270                 control |= 0x400000;
4271
4272         amdgpu_ring_write(ring, header);
4273         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
4274         amdgpu_ring_write(ring,
4275 #ifdef __BIG_ENDIAN
4276                 (2 << 0) |
4277 #endif
4278                 lower_32_bits(ib->gpu_addr));
4279         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
4280         amdgpu_ring_write(ring, control);
4281 }
4282
4283 static void gfx_v12_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
4284                                            struct amdgpu_job *job,
4285                                            struct amdgpu_ib *ib,
4286                                            uint32_t flags)
4287 {
4288         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
4289         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
4290
4291         if (ring->is_mes_queue)
4292                 /* inherit vmid from mqd */
4293                 control |= 0x40000000;
4294
4295         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
4296         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
4297         amdgpu_ring_write(ring,
4298 #ifdef __BIG_ENDIAN
4299                                 (2 << 0) |
4300 #endif
4301                                 lower_32_bits(ib->gpu_addr));
4302         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
4303         amdgpu_ring_write(ring, control);
4304 }
4305
4306 static void gfx_v12_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
4307                                      u64 seq, unsigned flags)
4308 {
4309         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
4310         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
4311
4312         /* RELEASE_MEM - flush caches, send int */
4313         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
4314         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
4315                                  PACKET3_RELEASE_MEM_GCR_GL2_WB |
4316                                  PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
4317                                  PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
4318                                  PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
4319         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
4320                                  PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
4321
4322         /*
4323          * the address should be Qword aligned if 64bit write, Dword
4324          * aligned if only send 32bit data low (discard data high)
4325          */
4326         if (write64bit)
4327                 BUG_ON(addr & 0x7);
4328         else
4329                 BUG_ON(addr & 0x3);
4330         amdgpu_ring_write(ring, lower_32_bits(addr));
4331         amdgpu_ring_write(ring, upper_32_bits(addr));
4332         amdgpu_ring_write(ring, lower_32_bits(seq));
4333         amdgpu_ring_write(ring, upper_32_bits(seq));
4334         amdgpu_ring_write(ring, ring->is_mes_queue ?
4335                          (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0);
4336 }
4337
4338 static void gfx_v12_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
4339 {
4340         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
4341         uint32_t seq = ring->fence_drv.sync_seq;
4342         uint64_t addr = ring->fence_drv.gpu_addr;
4343
4344         gfx_v12_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
4345                                upper_32_bits(addr), seq, 0xffffffff, 4);
4346 }
4347
4348 static void gfx_v12_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
4349                                    uint16_t pasid, uint32_t flush_type,
4350                                    bool all_hub, uint8_t dst_sel)
4351 {
4352         amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
4353         amdgpu_ring_write(ring,
4354                           PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
4355                           PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
4356                           PACKET3_INVALIDATE_TLBS_PASID(pasid) |
4357                           PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
4358 }
4359
4360 static void gfx_v12_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
4361                                          unsigned vmid, uint64_t pd_addr)
4362 {
4363         if (ring->is_mes_queue)
4364                 gfx_v12_0_ring_invalidate_tlbs(ring, 0, 0, false, 0);
4365         else
4366                 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
4367
4368         /* compute doesn't have PFP */
4369         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
4370                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4371                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4372                 amdgpu_ring_write(ring, 0x0);
4373         }
4374 }
4375
4376 static void gfx_v12_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
4377                                           u64 seq, unsigned int flags)
4378 {
4379         struct amdgpu_device *adev = ring->adev;
4380
4381         /* we only allocate 32bit for each seq wb address */
4382         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
4383
4384         /* write fence seq to the "addr" */
4385         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4386         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4387                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
4388         amdgpu_ring_write(ring, lower_32_bits(addr));
4389         amdgpu_ring_write(ring, upper_32_bits(addr));
4390         amdgpu_ring_write(ring, lower_32_bits(seq));
4391
4392         if (flags & AMDGPU_FENCE_FLAG_INT) {
4393                 /* set register to trigger INT */
4394                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4395                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4396                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
4397                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
4398                 amdgpu_ring_write(ring, 0);
4399                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
4400         }
4401 }
4402
4403 static void gfx_v12_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
4404                                          uint32_t flags)
4405 {
4406         uint32_t dw2 = 0;
4407
4408         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
4409         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
4410                 /* set load_global_config & load_global_uconfig */
4411                 dw2 |= 0x8001;
4412                 /* set load_cs_sh_regs */
4413                 dw2 |= 0x01000000;
4414                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
4415                 dw2 |= 0x10002;
4416         }
4417
4418         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4419         amdgpu_ring_write(ring, dw2);
4420         amdgpu_ring_write(ring, 0);
4421 }
4422
4423 static unsigned gfx_v12_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
4424                                                    uint64_t addr)
4425 {
4426         unsigned ret;
4427
4428         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
4429         amdgpu_ring_write(ring, lower_32_bits(addr));
4430         amdgpu_ring_write(ring, upper_32_bits(addr));
4431         /* discard following DWs if *cond_exec_gpu_addr==0 */
4432         amdgpu_ring_write(ring, 0);
4433         ret = ring->wptr & ring->buf_mask;
4434         /* patch dummy value later */
4435         amdgpu_ring_write(ring, 0);
4436
4437         return ret;
4438 }
4439
4440 static int gfx_v12_0_ring_preempt_ib(struct amdgpu_ring *ring)
4441 {
4442         int i, r = 0;
4443         struct amdgpu_device *adev = ring->adev;
4444         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
4445         struct amdgpu_ring *kiq_ring = &kiq->ring;
4446         unsigned long flags;
4447
4448         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
4449                 return -EINVAL;
4450
4451         spin_lock_irqsave(&kiq->ring_lock, flags);
4452
4453         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
4454                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4455                 return -ENOMEM;
4456         }
4457
4458         /* assert preemption condition */
4459         amdgpu_ring_set_preempt_cond_exec(ring, false);
4460
4461         /* assert IB preemption, emit the trailing fence */
4462         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
4463                                    ring->trail_fence_gpu_addr,
4464                                    ++ring->trail_seq);
4465         amdgpu_ring_commit(kiq_ring);
4466
4467         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4468
4469         /* poll the trailing fence */
4470         for (i = 0; i < adev->usec_timeout; i++) {
4471                 if (ring->trail_seq ==
4472                     le32_to_cpu(*(ring->trail_fence_cpu_addr)))
4473                         break;
4474                 udelay(1);
4475         }
4476
4477         if (i >= adev->usec_timeout) {
4478                 r = -EINVAL;
4479                 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
4480         }
4481
4482         /* deassert preemption condition */
4483         amdgpu_ring_set_preempt_cond_exec(ring, true);
4484         return r;
4485 }
4486
4487 static void gfx_v12_0_ring_emit_frame_cntl(struct amdgpu_ring *ring,
4488                                            bool start,
4489                                            bool secure)
4490 {
4491         uint32_t v = secure ? FRAME_TMZ : 0;
4492
4493         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
4494         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
4495 }
4496
4497 static void gfx_v12_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
4498                                      uint32_t reg_val_offs)
4499 {
4500         struct amdgpu_device *adev = ring->adev;
4501
4502         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4503         amdgpu_ring_write(ring, 0 |     /* src: register*/
4504                                 (5 << 8) |      /* dst: memory */
4505                                 (1 << 20));     /* write confirm */
4506         amdgpu_ring_write(ring, reg);
4507         amdgpu_ring_write(ring, 0);
4508         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4509                                 reg_val_offs * 4));
4510         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4511                                 reg_val_offs * 4));
4512 }
4513
4514 static void gfx_v12_0_ring_emit_wreg(struct amdgpu_ring *ring,
4515                                      uint32_t reg,
4516                                      uint32_t val)
4517 {
4518         uint32_t cmd = 0;
4519
4520         switch (ring->funcs->type) {
4521         case AMDGPU_RING_TYPE_GFX:
4522                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
4523                 break;
4524         case AMDGPU_RING_TYPE_KIQ:
4525                 cmd = (1 << 16); /* no inc addr */
4526                 break;
4527         default:
4528                 cmd = WR_CONFIRM;
4529                 break;
4530         }
4531         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4532         amdgpu_ring_write(ring, cmd);
4533         amdgpu_ring_write(ring, reg);
4534         amdgpu_ring_write(ring, 0);
4535         amdgpu_ring_write(ring, val);
4536 }
4537
4538 static void gfx_v12_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
4539                                         uint32_t val, uint32_t mask)
4540 {
4541         gfx_v12_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
4542 }
4543
4544 static void gfx_v12_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
4545                                                    uint32_t reg0, uint32_t reg1,
4546                                                    uint32_t ref, uint32_t mask)
4547 {
4548         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
4549
4550         gfx_v12_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
4551                                ref, mask, 0x20);
4552 }
4553
4554 static void gfx_v12_0_ring_soft_recovery(struct amdgpu_ring *ring,
4555                                          unsigned vmid)
4556 {
4557         struct amdgpu_device *adev = ring->adev;
4558         uint32_t value = 0;
4559
4560         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
4561         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
4562         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
4563         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
4564         WREG32_SOC15(GC, 0, regSQ_CMD, value);
4565 }
4566
4567 static void
4568 gfx_v12_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
4569                                       uint32_t me, uint32_t pipe,
4570                                       enum amdgpu_interrupt_state state)
4571 {
4572         uint32_t cp_int_cntl, cp_int_cntl_reg;
4573
4574         if (!me) {
4575                 switch (pipe) {
4576                 case 0:
4577                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
4578                         break;
4579                 default:
4580                         DRM_DEBUG("invalid pipe %d\n", pipe);
4581                         return;
4582                 }
4583         } else {
4584                 DRM_DEBUG("invalid me %d\n", me);
4585                 return;
4586         }
4587
4588         switch (state) {
4589         case AMDGPU_IRQ_STATE_DISABLE:
4590                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
4591                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
4592                                             TIME_STAMP_INT_ENABLE, 0);
4593                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
4594                                             GENERIC0_INT_ENABLE, 0);
4595                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
4596                 break;
4597         case AMDGPU_IRQ_STATE_ENABLE:
4598                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
4599                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
4600                                             TIME_STAMP_INT_ENABLE, 1);
4601                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
4602                                             GENERIC0_INT_ENABLE, 1);
4603                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
4604                 break;
4605         default:
4606                 break;
4607         }
4608 }
4609
4610 static void gfx_v12_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
4611                                                      int me, int pipe,
4612                                                      enum amdgpu_interrupt_state state)
4613 {
4614         u32 mec_int_cntl, mec_int_cntl_reg;
4615
4616         /*
4617          * amdgpu controls only the first MEC. That's why this function only
4618          * handles the setting of interrupts for this specific MEC. All other
4619          * pipes' interrupts are set by amdkfd.
4620          */
4621
4622         if (me == 1) {
4623                 switch (pipe) {
4624                 case 0:
4625                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
4626                         break;
4627                 case 1:
4628                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
4629                         break;
4630                 default:
4631                         DRM_DEBUG("invalid pipe %d\n", pipe);
4632                         return;
4633                 }
4634         } else {
4635                 DRM_DEBUG("invalid me %d\n", me);
4636                 return;
4637         }
4638
4639         switch (state) {
4640         case AMDGPU_IRQ_STATE_DISABLE:
4641                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
4642                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4643                                              TIME_STAMP_INT_ENABLE, 0);
4644                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4645                                              GENERIC0_INT_ENABLE, 0);
4646                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
4647                 break;
4648         case AMDGPU_IRQ_STATE_ENABLE:
4649                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
4650                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4651                                              TIME_STAMP_INT_ENABLE, 1);
4652                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4653                                              GENERIC0_INT_ENABLE, 1);
4654                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
4655                 break;
4656         default:
4657                 break;
4658         }
4659 }
4660
4661 static int gfx_v12_0_set_eop_interrupt_state(struct amdgpu_device *adev,
4662                                             struct amdgpu_irq_src *src,
4663                                             unsigned type,
4664                                             enum amdgpu_interrupt_state state)
4665 {
4666         switch (type) {
4667         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
4668                 gfx_v12_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
4669                 break;
4670         case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
4671                 gfx_v12_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
4672                 break;
4673         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
4674                 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
4675                 break;
4676         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
4677                 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
4678                 break;
4679         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
4680                 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
4681                 break;
4682         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
4683                 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
4684                 break;
4685         default:
4686                 break;
4687         }
4688         return 0;
4689 }
4690
4691 static int gfx_v12_0_eop_irq(struct amdgpu_device *adev,
4692                              struct amdgpu_irq_src *source,
4693                              struct amdgpu_iv_entry *entry)
4694 {
4695         int i;
4696         u8 me_id, pipe_id, queue_id;
4697         struct amdgpu_ring *ring;
4698         uint32_t mes_queue_id = entry->src_data[0];
4699
4700         DRM_DEBUG("IH: CP EOP\n");
4701
4702         if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
4703                 struct amdgpu_mes_queue *queue;
4704
4705                 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
4706
4707                 spin_lock(&adev->mes.queue_id_lock);
4708                 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
4709                 if (queue) {
4710                         DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
4711                         amdgpu_fence_process(queue->ring);
4712                 }
4713                 spin_unlock(&adev->mes.queue_id_lock);
4714         } else {
4715                 me_id = (entry->ring_id & 0x0c) >> 2;
4716                 pipe_id = (entry->ring_id & 0x03) >> 0;
4717                 queue_id = (entry->ring_id & 0x70) >> 4;
4718
4719                 switch (me_id) {
4720                 case 0:
4721                         if (pipe_id == 0)
4722                                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
4723                         else
4724                                 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
4725                         break;
4726                 case 1:
4727                 case 2:
4728                         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4729                                 ring = &adev->gfx.compute_ring[i];
4730                                 /* Per-queue interrupt is supported for MEC starting from VI.
4731                                  * The interrupt can only be enabled/disabled per pipe instead
4732                                  * of per queue.
4733                                  */
4734                                 if ((ring->me == me_id) &&
4735                                     (ring->pipe == pipe_id) &&
4736                                     (ring->queue == queue_id))
4737                                         amdgpu_fence_process(ring);
4738                         }
4739                         break;
4740                 }
4741         }
4742
4743         return 0;
4744 }
4745
4746 static int gfx_v12_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
4747                                               struct amdgpu_irq_src *source,
4748                                               unsigned type,
4749                                               enum amdgpu_interrupt_state state)
4750 {
4751         switch (state) {
4752         case AMDGPU_IRQ_STATE_DISABLE:
4753         case AMDGPU_IRQ_STATE_ENABLE:
4754                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
4755                                       PRIV_REG_INT_ENABLE,
4756                                       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
4757                 break;
4758         default:
4759                 break;
4760         }
4761
4762         return 0;
4763 }
4764
4765 static int gfx_v12_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
4766                                                struct amdgpu_irq_src *source,
4767                                                unsigned type,
4768                                                enum amdgpu_interrupt_state state)
4769 {
4770         switch (state) {
4771         case AMDGPU_IRQ_STATE_DISABLE:
4772         case AMDGPU_IRQ_STATE_ENABLE:
4773                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
4774                                PRIV_INSTR_INT_ENABLE,
4775                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
4776                 break;
4777         default:
4778                 break;
4779         }
4780
4781         return 0;
4782 }
4783
4784 static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev,
4785                                         struct amdgpu_iv_entry *entry)
4786 {
4787         u8 me_id, pipe_id, queue_id;
4788         struct amdgpu_ring *ring;
4789         int i;
4790
4791         me_id = (entry->ring_id & 0x0c) >> 2;
4792         pipe_id = (entry->ring_id & 0x03) >> 0;
4793         queue_id = (entry->ring_id & 0x70) >> 4;
4794
4795         switch (me_id) {
4796         case 0:
4797                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4798                         ring = &adev->gfx.gfx_ring[i];
4799                         /* we only enabled 1 gfx queue per pipe for now */
4800                         if (ring->me == me_id && ring->pipe == pipe_id)
4801                                 drm_sched_fault(&ring->sched);
4802                 }
4803                 break;
4804         case 1:
4805         case 2:
4806                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4807                         ring = &adev->gfx.compute_ring[i];
4808                         if (ring->me == me_id && ring->pipe == pipe_id &&
4809                             ring->queue == queue_id)
4810                                 drm_sched_fault(&ring->sched);
4811                 }
4812                 break;
4813         default:
4814                 BUG();
4815                 break;
4816         }
4817 }
4818
4819 static int gfx_v12_0_priv_reg_irq(struct amdgpu_device *adev,
4820                                   struct amdgpu_irq_src *source,
4821                                   struct amdgpu_iv_entry *entry)
4822 {
4823         DRM_ERROR("Illegal register access in command stream\n");
4824         gfx_v12_0_handle_priv_fault(adev, entry);
4825         return 0;
4826 }
4827
4828 static int gfx_v12_0_priv_inst_irq(struct amdgpu_device *adev,
4829                                    struct amdgpu_irq_src *source,
4830                                    struct amdgpu_iv_entry *entry)
4831 {
4832         DRM_ERROR("Illegal instruction in command stream\n");
4833         gfx_v12_0_handle_priv_fault(adev, entry);
4834         return 0;
4835 }
4836
4837 static void gfx_v12_0_emit_mem_sync(struct amdgpu_ring *ring)
4838 {
4839         const unsigned int gcr_cntl =
4840                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
4841                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
4842                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
4843                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
4844                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
4845                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
4846                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
4847                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
4848
4849         /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
4850         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
4851         amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
4852         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
4853         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
4854         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
4855         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
4856         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
4857         amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
4858 }
4859
4860 static void gfx_v12_ip_print(void *handle, struct drm_printer *p)
4861 {
4862         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4863         uint32_t i, j, k, reg, index = 0;
4864         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_0);
4865
4866         if (!adev->gfx.ip_dump_core)
4867                 return;
4868
4869         for (i = 0; i < reg_count; i++)
4870                 drm_printf(p, "%-50s \t 0x%08x\n",
4871                            gc_reg_list_12_0[i].reg_name,
4872                            adev->gfx.ip_dump_core[i]);
4873
4874         /* print compute queue registers for all instances */
4875         if (!adev->gfx.ip_dump_compute_queues)
4876                 return;
4877
4878         reg_count = ARRAY_SIZE(gc_cp_reg_list_12);
4879         drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
4880                    adev->gfx.mec.num_mec,
4881                    adev->gfx.mec.num_pipe_per_mec,
4882                    adev->gfx.mec.num_queue_per_pipe);
4883
4884         for (i = 0; i < adev->gfx.mec.num_mec; i++) {
4885                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
4886                         for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
4887                                 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
4888                                 for (reg = 0; reg < reg_count; reg++) {
4889                                         drm_printf(p, "%-50s \t 0x%08x\n",
4890                                                    gc_cp_reg_list_12[reg].reg_name,
4891                                                    adev->gfx.ip_dump_compute_queues[index + reg]);
4892                                 }
4893                                 index += reg_count;
4894                         }
4895                 }
4896         }
4897
4898         /* print gfx queue registers for all instances */
4899         if (!adev->gfx.ip_dump_gfx_queues)
4900                 return;
4901
4902         index = 0;
4903         reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_12);
4904         drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n",
4905                    adev->gfx.me.num_me,
4906                    adev->gfx.me.num_pipe_per_me,
4907                    adev->gfx.me.num_queue_per_pipe);
4908
4909         for (i = 0; i < adev->gfx.me.num_me; i++) {
4910                 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
4911                         for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
4912                                 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k);
4913                                 for (reg = 0; reg < reg_count; reg++) {
4914                                         drm_printf(p, "%-50s \t 0x%08x\n",
4915                                                    gc_gfx_queue_reg_list_12[reg].reg_name,
4916                                                    adev->gfx.ip_dump_gfx_queues[index + reg]);
4917                                 }
4918                                 index += reg_count;
4919                         }
4920                 }
4921         }
4922 }
4923
4924 static void gfx_v12_ip_dump(void *handle)
4925 {
4926         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4927         uint32_t i, j, k, reg, index = 0;
4928         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_0);
4929
4930         if (!adev->gfx.ip_dump_core)
4931                 return;
4932
4933         amdgpu_gfx_off_ctrl(adev, false);
4934         for (i = 0; i < reg_count; i++)
4935                 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_12_0[i]));
4936         amdgpu_gfx_off_ctrl(adev, true);
4937
4938         /* dump compute queue registers for all instances */
4939         if (!adev->gfx.ip_dump_compute_queues)
4940                 return;
4941
4942         reg_count = ARRAY_SIZE(gc_cp_reg_list_12);
4943         amdgpu_gfx_off_ctrl(adev, false);
4944         mutex_lock(&adev->srbm_mutex);
4945         for (i = 0; i < adev->gfx.mec.num_mec; i++) {
4946                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
4947                         for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
4948                                 /* ME0 is for GFX so start from 1 for CP */
4949                                 soc24_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0);
4950                                 for (reg = 0; reg < reg_count; reg++) {
4951                                         adev->gfx.ip_dump_compute_queues[index + reg] =
4952                                                 RREG32(SOC15_REG_ENTRY_OFFSET(
4953                                                         gc_cp_reg_list_12[reg]));
4954                                 }
4955                                 index += reg_count;
4956                         }
4957                 }
4958         }
4959         soc24_grbm_select(adev, 0, 0, 0, 0);
4960         mutex_unlock(&adev->srbm_mutex);
4961         amdgpu_gfx_off_ctrl(adev, true);
4962
4963         /* dump gfx queue registers for all instances */
4964         if (!adev->gfx.ip_dump_gfx_queues)
4965                 return;
4966
4967         index = 0;
4968         reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_12);
4969         amdgpu_gfx_off_ctrl(adev, false);
4970         mutex_lock(&adev->srbm_mutex);
4971         for (i = 0; i < adev->gfx.me.num_me; i++) {
4972                 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
4973                         for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
4974                                 soc24_grbm_select(adev, i, j, k, 0);
4975
4976                                 for (reg = 0; reg < reg_count; reg++) {
4977                                         adev->gfx.ip_dump_gfx_queues[index + reg] =
4978                                                 RREG32(SOC15_REG_ENTRY_OFFSET(
4979                                                         gc_gfx_queue_reg_list_12[reg]));
4980                                 }
4981                                 index += reg_count;
4982                         }
4983                 }
4984         }
4985         soc24_grbm_select(adev, 0, 0, 0, 0);
4986         mutex_unlock(&adev->srbm_mutex);
4987         amdgpu_gfx_off_ctrl(adev, true);
4988 }
4989
4990 static const struct amd_ip_funcs gfx_v12_0_ip_funcs = {
4991         .name = "gfx_v12_0",
4992         .early_init = gfx_v12_0_early_init,
4993         .late_init = gfx_v12_0_late_init,
4994         .sw_init = gfx_v12_0_sw_init,
4995         .sw_fini = gfx_v12_0_sw_fini,
4996         .hw_init = gfx_v12_0_hw_init,
4997         .hw_fini = gfx_v12_0_hw_fini,
4998         .suspend = gfx_v12_0_suspend,
4999         .resume = gfx_v12_0_resume,
5000         .is_idle = gfx_v12_0_is_idle,
5001         .wait_for_idle = gfx_v12_0_wait_for_idle,
5002         .set_clockgating_state = gfx_v12_0_set_clockgating_state,
5003         .set_powergating_state = gfx_v12_0_set_powergating_state,
5004         .get_clockgating_state = gfx_v12_0_get_clockgating_state,
5005         .dump_ip_state = gfx_v12_ip_dump,
5006         .print_ip_state = gfx_v12_ip_print,
5007 };
5008
5009 static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_gfx = {
5010         .type = AMDGPU_RING_TYPE_GFX,
5011         .align_mask = 0xff,
5012         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
5013         .support_64bit_ptrs = true,
5014         .secure_submission_supported = true,
5015         .get_rptr = gfx_v12_0_ring_get_rptr_gfx,
5016         .get_wptr = gfx_v12_0_ring_get_wptr_gfx,
5017         .set_wptr = gfx_v12_0_ring_set_wptr_gfx,
5018         .emit_frame_size = /* totally 242 maximum if 16 IBs */
5019                 5 + /* COND_EXEC */
5020                 7 + /* PIPELINE_SYNC */
5021                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
5022                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
5023                 2 + /* VM_FLUSH */
5024                 8 + /* FENCE for VM_FLUSH */
5025                 5 + /* COND_EXEC */
5026                 7 + /* HDP_flush */
5027                 4 + /* VGT_flush */
5028                 31 + /* DE_META */
5029                 3 + /* CNTX_CTRL */
5030                 5 + /* HDP_INVL */
5031                 8 + 8 + /* FENCE x2 */
5032                 8, /* gfx_v12_0_emit_mem_sync */
5033         .emit_ib_size = 4, /* gfx_v12_0_ring_emit_ib_gfx */
5034         .emit_ib = gfx_v12_0_ring_emit_ib_gfx,
5035         .emit_fence = gfx_v12_0_ring_emit_fence,
5036         .emit_pipeline_sync = gfx_v12_0_ring_emit_pipeline_sync,
5037         .emit_vm_flush = gfx_v12_0_ring_emit_vm_flush,
5038         .emit_hdp_flush = gfx_v12_0_ring_emit_hdp_flush,
5039         .test_ring = gfx_v12_0_ring_test_ring,
5040         .test_ib = gfx_v12_0_ring_test_ib,
5041         .insert_nop = amdgpu_ring_insert_nop,
5042         .pad_ib = amdgpu_ring_generic_pad_ib,
5043         .emit_cntxcntl = gfx_v12_0_ring_emit_cntxcntl,
5044         .init_cond_exec = gfx_v12_0_ring_emit_init_cond_exec,
5045         .preempt_ib = gfx_v12_0_ring_preempt_ib,
5046         .emit_frame_cntl = gfx_v12_0_ring_emit_frame_cntl,
5047         .emit_wreg = gfx_v12_0_ring_emit_wreg,
5048         .emit_reg_wait = gfx_v12_0_ring_emit_reg_wait,
5049         .emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait,
5050         .soft_recovery = gfx_v12_0_ring_soft_recovery,
5051         .emit_mem_sync = gfx_v12_0_emit_mem_sync,
5052 };
5053
5054 static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_compute = {
5055         .type = AMDGPU_RING_TYPE_COMPUTE,
5056         .align_mask = 0xff,
5057         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
5058         .support_64bit_ptrs = true,
5059         .get_rptr = gfx_v12_0_ring_get_rptr_compute,
5060         .get_wptr = gfx_v12_0_ring_get_wptr_compute,
5061         .set_wptr = gfx_v12_0_ring_set_wptr_compute,
5062         .emit_frame_size =
5063                 7 + /* gfx_v12_0_ring_emit_hdp_flush */
5064                 5 + /* hdp invalidate */
5065                 7 + /* gfx_v12_0_ring_emit_pipeline_sync */
5066                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
5067                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
5068                 2 + /* gfx_v12_0_ring_emit_vm_flush */
5069                 8 + 8 + 8 + /* gfx_v12_0_ring_emit_fence x3 for user fence, vm fence */
5070                 8, /* gfx_v12_0_emit_mem_sync */
5071         .emit_ib_size = 7, /* gfx_v12_0_ring_emit_ib_compute */
5072         .emit_ib = gfx_v12_0_ring_emit_ib_compute,
5073         .emit_fence = gfx_v12_0_ring_emit_fence,
5074         .emit_pipeline_sync = gfx_v12_0_ring_emit_pipeline_sync,
5075         .emit_vm_flush = gfx_v12_0_ring_emit_vm_flush,
5076         .emit_hdp_flush = gfx_v12_0_ring_emit_hdp_flush,
5077         .test_ring = gfx_v12_0_ring_test_ring,
5078         .test_ib = gfx_v12_0_ring_test_ib,
5079         .insert_nop = amdgpu_ring_insert_nop,
5080         .pad_ib = amdgpu_ring_generic_pad_ib,
5081         .emit_wreg = gfx_v12_0_ring_emit_wreg,
5082         .emit_reg_wait = gfx_v12_0_ring_emit_reg_wait,
5083         .emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait,
5084         .emit_mem_sync = gfx_v12_0_emit_mem_sync,
5085 };
5086
5087 static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_kiq = {
5088         .type = AMDGPU_RING_TYPE_KIQ,
5089         .align_mask = 0xff,
5090         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
5091         .support_64bit_ptrs = true,
5092         .get_rptr = gfx_v12_0_ring_get_rptr_compute,
5093         .get_wptr = gfx_v12_0_ring_get_wptr_compute,
5094         .set_wptr = gfx_v12_0_ring_set_wptr_compute,
5095         .emit_frame_size =
5096                 7 + /* gfx_v12_0_ring_emit_hdp_flush */
5097                 5 + /*hdp invalidate */
5098                 7 + /* gfx_v12_0_ring_emit_pipeline_sync */
5099                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
5100                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
5101                 2 + /* gfx_v12_0_ring_emit_vm_flush */
5102                 8 + 8 + 8, /* gfx_v12_0_ring_emit_fence_kiq x3 for user fence, vm fence */
5103         .emit_ib_size = 7, /* gfx_v12_0_ring_emit_ib_compute */
5104         .emit_ib = gfx_v12_0_ring_emit_ib_compute,
5105         .emit_fence = gfx_v12_0_ring_emit_fence_kiq,
5106         .test_ring = gfx_v12_0_ring_test_ring,
5107         .test_ib = gfx_v12_0_ring_test_ib,
5108         .insert_nop = amdgpu_ring_insert_nop,
5109         .pad_ib = amdgpu_ring_generic_pad_ib,
5110         .emit_rreg = gfx_v12_0_ring_emit_rreg,
5111         .emit_wreg = gfx_v12_0_ring_emit_wreg,
5112         .emit_reg_wait = gfx_v12_0_ring_emit_reg_wait,
5113         .emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait,
5114 };
5115
5116 static void gfx_v12_0_set_ring_funcs(struct amdgpu_device *adev)
5117 {
5118         int i;
5119
5120         adev->gfx.kiq[0].ring.funcs = &gfx_v12_0_ring_funcs_kiq;
5121
5122         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
5123                 adev->gfx.gfx_ring[i].funcs = &gfx_v12_0_ring_funcs_gfx;
5124
5125         for (i = 0; i < adev->gfx.num_compute_rings; i++)
5126                 adev->gfx.compute_ring[i].funcs = &gfx_v12_0_ring_funcs_compute;
5127 }
5128
5129 static const struct amdgpu_irq_src_funcs gfx_v12_0_eop_irq_funcs = {
5130         .set = gfx_v12_0_set_eop_interrupt_state,
5131         .process = gfx_v12_0_eop_irq,
5132 };
5133
5134 static const struct amdgpu_irq_src_funcs gfx_v12_0_priv_reg_irq_funcs = {
5135         .set = gfx_v12_0_set_priv_reg_fault_state,
5136         .process = gfx_v12_0_priv_reg_irq,
5137 };
5138
5139 static const struct amdgpu_irq_src_funcs gfx_v12_0_priv_inst_irq_funcs = {
5140         .set = gfx_v12_0_set_priv_inst_fault_state,
5141         .process = gfx_v12_0_priv_inst_irq,
5142 };
5143
5144 static void gfx_v12_0_set_irq_funcs(struct amdgpu_device *adev)
5145 {
5146         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
5147         adev->gfx.eop_irq.funcs = &gfx_v12_0_eop_irq_funcs;
5148
5149         adev->gfx.priv_reg_irq.num_types = 1;
5150         adev->gfx.priv_reg_irq.funcs = &gfx_v12_0_priv_reg_irq_funcs;
5151
5152         adev->gfx.priv_inst_irq.num_types = 1;
5153         adev->gfx.priv_inst_irq.funcs = &gfx_v12_0_priv_inst_irq_funcs;
5154 }
5155
5156 static void gfx_v12_0_set_imu_funcs(struct amdgpu_device *adev)
5157 {
5158         if (adev->flags & AMD_IS_APU)
5159                 adev->gfx.imu.mode = MISSION_MODE;
5160         else
5161                 adev->gfx.imu.mode = DEBUG_MODE;
5162
5163         adev->gfx.imu.funcs = &gfx_v12_0_imu_funcs;
5164 }
5165
5166 static void gfx_v12_0_set_rlc_funcs(struct amdgpu_device *adev)
5167 {
5168         adev->gfx.rlc.funcs = &gfx_v12_0_rlc_funcs;
5169 }
5170
5171 static void gfx_v12_0_set_mqd_funcs(struct amdgpu_device *adev)
5172 {
5173         /* set gfx eng mqd */
5174         adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
5175                 sizeof(struct v12_gfx_mqd);
5176         adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
5177                 gfx_v12_0_gfx_mqd_init;
5178         /* set compute eng mqd */
5179         adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
5180                 sizeof(struct v12_compute_mqd);
5181         adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
5182                 gfx_v12_0_compute_mqd_init;
5183 }
5184
5185 static void gfx_v12_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
5186                                                           u32 bitmap)
5187 {
5188         u32 data;
5189
5190         if (!bitmap)
5191                 return;
5192
5193         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
5194         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
5195
5196         WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
5197 }
5198
5199 static u32 gfx_v12_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
5200 {
5201         u32 data, wgp_bitmask;
5202         data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
5203         data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
5204
5205         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
5206         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
5207
5208         wgp_bitmask =
5209                 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
5210
5211         return (~data) & wgp_bitmask;
5212 }
5213
5214 static u32 gfx_v12_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
5215 {
5216         u32 wgp_idx, wgp_active_bitmap;
5217         u32 cu_bitmap_per_wgp, cu_active_bitmap;
5218
5219         wgp_active_bitmap = gfx_v12_0_get_wgp_active_bitmap_per_sh(adev);
5220         cu_active_bitmap = 0;
5221
5222         for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
5223                 /* if there is one WGP enabled, it means 2 CUs will be enabled */
5224                 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
5225                 if (wgp_active_bitmap & (1 << wgp_idx))
5226                         cu_active_bitmap |= cu_bitmap_per_wgp;
5227         }
5228
5229         return cu_active_bitmap;
5230 }
5231
5232 static int gfx_v12_0_get_cu_info(struct amdgpu_device *adev,
5233                                  struct amdgpu_cu_info *cu_info)
5234 {
5235         int i, j, k, counter, active_cu_number = 0;
5236         u32 mask, bitmap;
5237         unsigned disable_masks[8 * 2];
5238
5239         if (!adev || !cu_info)
5240                 return -EINVAL;
5241
5242         amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
5243
5244         mutex_lock(&adev->grbm_idx_mutex);
5245         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
5246                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
5247                         bitmap = i * adev->gfx.config.max_sh_per_se + j;
5248                         if (!((gfx_v12_0_get_sa_active_bitmap(adev) >> bitmap) & 1))
5249                                 continue;
5250                         mask = 1;
5251                         counter = 0;
5252                         gfx_v12_0_select_se_sh(adev, i, j, 0xffffffff, 0);
5253                         if (i < 8 && j < 2)
5254                                 gfx_v12_0_set_user_wgp_inactive_bitmap_per_sh(
5255                                         adev, disable_masks[i * 2 + j]);
5256                         bitmap = gfx_v12_0_get_cu_active_bitmap_per_sh(adev);
5257
5258                         /**
5259                          * GFX12 could support more than 4 SEs, while the bitmap
5260                          * in cu_info struct is 4x4 and ioctl interface struct
5261                          * drm_amdgpu_info_device should keep stable.
5262                          * So we use last two columns of bitmap to store cu mask for
5263                          * SEs 4 to 7, the layout of the bitmap is as below:
5264                          *    SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
5265                          *    SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
5266                          *    SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
5267                          *    SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
5268                          *    SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
5269                          *    SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
5270                          *    SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
5271                          *    SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
5272                          */
5273                         cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap;
5274
5275                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
5276                                 if (bitmap & mask)
5277                                         counter++;
5278
5279                                 mask <<= 1;
5280                         }
5281                         active_cu_number += counter;
5282                 }
5283         }
5284         gfx_v12_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
5285         mutex_unlock(&adev->grbm_idx_mutex);
5286
5287         cu_info->number = active_cu_number;
5288         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
5289
5290         return 0;
5291 }
5292
5293 const struct amdgpu_ip_block_version gfx_v12_0_ip_block = {
5294         .type = AMD_IP_BLOCK_TYPE_GFX,
5295         .major = 12,
5296         .minor = 0,
5297         .rev = 0,
5298         .funcs = &gfx_v12_0_ip_funcs,
5299 };