drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c

   1 /*
   2  * Copyright 2021 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <linux/delay.h>
  24 #include <linux/kernel.h>
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_gfx.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_smu.h"
  32 #include "amdgpu_atomfirmware.h"
  33 #include "imu_v11_0.h"
  34 #include "soc21.h"
  35 #include "nvd.h"
  36
  37 #include "gc/gc_11_0_0_offset.h"
  38 #include "gc/gc_11_0_0_sh_mask.h"
  39 #include "smuio/smuio_13_0_6_offset.h"
  40 #include "smuio/smuio_13_0_6_sh_mask.h"
  41 #include "navi10_enum.h"
  42 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
  43
  44 #include "soc15.h"
  45 #include "soc15d.h"
  46 #include "clearstate_gfx11.h"
  47 #include "v11_structs.h"
  48 #include "gfx_v11_0.h"
  49 #include "gfx_v11_0_cleaner_shader.h"
  50 #include "gfx_v11_0_3.h"
  51 #include "nbio_v4_3.h"
  52 #include "mes_v11_0.h"
  53
  54 #define GFX11_NUM_GFX_RINGS             1
  55 #define GFX11_MEC_HPD_SIZE      2048
  56
  57 #define RLCG_UCODE_LOADING_START_ADDRESS        0x00002000L
  58 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1        0x1388
  59
  60 #define regCGTT_WD_CLK_CTRL             0x5086
  61 #define regCGTT_WD_CLK_CTRL_BASE_IDX    1
  62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1   0x4e7e
  63 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX  1
  64 #define regPC_CONFIG_CNTL_1             0x194d
  65 #define regPC_CONFIG_CNTL_1_BASE_IDX    1
  66
  67 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin");
  68 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin");
  69 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin");
  70 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin");
  71 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin");
  72 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin");
  73 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin");
  74 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin");
  75 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin");
  76 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin");
  77 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin");
  78 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin");
  79 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin");
  80 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin");
  81 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin");
  82 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin");
  83 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin");
  84 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin");
  85 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin");
  86 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin");
  87 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin");
  88 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin");
  89 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin");
  90 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin");
  91 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin");
  92 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin");
  93 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin");
  94 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin");
  95 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin");
  96 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin");
  97 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin");
  98 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin");
  99 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin");
 100 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin");
 101
 102 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = {
 103         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
 104         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
 105         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3),
 106         SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
 107         SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
 108         SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3),
 109         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
 110         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
 111         SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
 112         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
 113         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
 114         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2),
 115         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2),
 116         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
 117         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
 118         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0),
 119         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE),
 120         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
 121         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR),
 122         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE),
 123         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR),
 124         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR),
 125         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE),
 126         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR),
 127         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR),
 128         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
 129         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ),
 130         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
 131         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
 132         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
 133         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO),
 134         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI),
 135         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ),
 136         SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
 137         SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
 138         SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
 139         SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT),
 140         SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT),
 141         SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS),
 142         SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2),
 143         SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS),
 144         SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS),
 145         SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
 146         SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES),
 147         SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS),
 148         SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
 149         SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL),
 150         SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS),
 151         SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
 152         SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
 153         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL),
 154         SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR),
 155         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR),
 156         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR),
 157         SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR),
 158         SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR),
 159         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
 160         /* cp header registers */
 161         SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
 162         SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
 163         SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
 164         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
 165         /* SE status registers */
 166         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
 167         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
 168         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2),
 169         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3),
 170         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4),
 171         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5)
 172 };
 173
 174 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = {
 175         /* compute registers */
 176         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
 177         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
 178         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
 179         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
 180         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
 181         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
 182         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
 183         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
 184         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
 185         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
 186         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
 187         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
 188         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
 189         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
 190         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
 191         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
 192         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
 193         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
 194         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
 195         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
 196         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
 197         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
 198         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
 199         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
 200         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
 201         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
 202         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
 203         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
 204         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
 205         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
 206         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
 207         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
 208         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
 209         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
 210         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
 211         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET),
 212         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT),
 213         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET),
 214         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS)
 215 };
 216
 217 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = {
 218         /* gfx queue registers */
 219         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE),
 220         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID),
 221         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY),
 222         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM),
 223         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE),
 224         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI),
 225         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET),
 226         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL),
 227         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR),
 228         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR),
 229         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI),
 230         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST),
 231         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED),
 232         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL),
 233         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0),
 234         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0),
 235         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR),
 236         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI),
 237         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO),
 238         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI),
 239         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
 240         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
 241         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
 242         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
 243         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ)
 244 };
 245
 246 static const struct soc15_reg_golden golden_settings_gc_11_0[] = {
 247         SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000)
 248 };
 249
 250 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =
 251 {
 252         SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010),
 253         SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010),
 254         SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200),
 255         SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988),
 256         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007),
 257         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008),
 258         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100),
 259         SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
 260         SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a)
 261 };
 262
 263 #define DEFAULT_SH_MEM_CONFIG \
 264         ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
 265          (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
 266          (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
 267
 268 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev);
 269 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev);
 270 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev);
 271 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev);
 272 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev);
 273 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev);
 274 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev);
 275 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
 276                                  struct amdgpu_cu_info *cu_info);
 277 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 278 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
 279                                    u32 sh_num, u32 instance, int xcc_id);
 280 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
 281
 282 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
 283 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
 284 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
 285                                      uint32_t val);
 286 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
 287 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
 288                                            uint16_t pasid, uint32_t flush_type,
 289                                            bool all_hub, uint8_t dst_sel);
 290 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
 291 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
 292 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
 293                                       bool enable);
 294
 295 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
 296 {
 297         struct amdgpu_device *adev = kiq_ring->adev;
 298         u64 shader_mc_addr;
 299
 300         /* Cleaner shader MC address */
 301         shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
 302
 303         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 304         amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
 305                           PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */
 306                           PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
 307         amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
 308         amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
 309         amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
 310         amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
 311         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 312         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 313 }
 314
 315 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 316                                  struct amdgpu_ring *ring)
 317 {
 318         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 319         uint64_t wptr_addr = ring->wptr_gpu_addr;
 320         uint32_t me = 0, eng_sel = 0;
 321
 322         switch (ring->funcs->type) {
 323         case AMDGPU_RING_TYPE_COMPUTE:
 324                 me = 1;
 325                 eng_sel = 0;
 326                 break;
 327         case AMDGPU_RING_TYPE_GFX:
 328                 me = 0;
 329                 eng_sel = 4;
 330                 break;
 331         case AMDGPU_RING_TYPE_MES:
 332                 me = 2;
 333                 eng_sel = 5;
 334                 break;
 335         default:
 336                 WARN_ON(1);
 337         }
 338
 339         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 340         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 341         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 342                           PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 343                           PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 344                           PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 345                           PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 346                           PACKET3_MAP_QUEUES_ME((me)) |
 347                           PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
 348                           PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
 349                           PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 350                           PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
 351         amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 352         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 353         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 354         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 355         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 356 }
 357
 358 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 359                                    struct amdgpu_ring *ring,
 360                                    enum amdgpu_unmap_queues_action action,
 361                                    u64 gpu_addr, u64 seq)
 362 {
 363         struct amdgpu_device *adev = kiq_ring->adev;
 364         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 365
 366         if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
 367                 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
 368                 return;
 369         }
 370
 371         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 372         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 373                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 374                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 375                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 376                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 377         amdgpu_ring_write(kiq_ring,
 378                   PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 379
 380         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 381                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 382                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 383                 amdgpu_ring_write(kiq_ring, seq);
 384         } else {
 385                 amdgpu_ring_write(kiq_ring, 0);
 386                 amdgpu_ring_write(kiq_ring, 0);
 387                 amdgpu_ring_write(kiq_ring, 0);
 388         }
 389 }
 390
 391 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring,
 392                                    struct amdgpu_ring *ring,
 393                                    u64 addr,
 394                                    u64 seq)
 395 {
 396         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 397
 398         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 399         amdgpu_ring_write(kiq_ring,
 400                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 401                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 402                           PACKET3_QUERY_STATUS_COMMAND(2));
 403         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 404                           PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 405                           PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 406         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 407         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 408         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 409         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 410 }
 411
 412 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 413                                 uint16_t pasid, uint32_t flush_type,
 414                                 bool all_hub)
 415 {
 416         gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
 417 }
 418
 419 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = {
 420         .kiq_set_resources = gfx11_kiq_set_resources,
 421         .kiq_map_queues = gfx11_kiq_map_queues,
 422         .kiq_unmap_queues = gfx11_kiq_unmap_queues,
 423         .kiq_query_status = gfx11_kiq_query_status,
 424         .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs,
 425         .set_resources_size = 8,
 426         .map_queues_size = 7,
 427         .unmap_queues_size = 6,
 428         .query_status_size = 7,
 429         .invalidate_tlbs_size = 2,
 430 };
 431
 432 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 433 {
 434         adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs;
 435 }
 436
 437 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)
 438 {
 439         if (amdgpu_sriov_vf(adev))
 440                 return;
 441
 442         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 443         case IP_VERSION(11, 0, 1):
 444         case IP_VERSION(11, 0, 4):
 445                 soc15_program_register_sequence(adev,
 446                                                 golden_settings_gc_11_0_1,
 447                                                 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));
 448                 break;
 449         default:
 450                 break;
 451         }
 452         soc15_program_register_sequence(adev,
 453                                         golden_settings_gc_11_0,
 454                                         (const u32)ARRAY_SIZE(golden_settings_gc_11_0));
 455
 456 }
 457
 458 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
 459                                        bool wc, uint32_t reg, uint32_t val)
 460 {
 461         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 462         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
 463                           WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
 464         amdgpu_ring_write(ring, reg);
 465         amdgpu_ring_write(ring, 0);
 466         amdgpu_ring_write(ring, val);
 467 }
 468
 469 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 470                                   int mem_space, int opt, uint32_t addr0,
 471                                   uint32_t addr1, uint32_t ref, uint32_t mask,
 472                                   uint32_t inv)
 473 {
 474         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 475         amdgpu_ring_write(ring,
 476                           /* memory (1) or register (0) */
 477                           (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 478                            WAIT_REG_MEM_OPERATION(opt) | /* wait */
 479                            WAIT_REG_MEM_FUNCTION(3) |  /* equal */
 480                            WAIT_REG_MEM_ENGINE(eng_sel)));
 481
 482         if (mem_space)
 483                 BUG_ON(addr0 & 0x3); /* Dword align */
 484         amdgpu_ring_write(ring, addr0);
 485         amdgpu_ring_write(ring, addr1);
 486         amdgpu_ring_write(ring, ref);
 487         amdgpu_ring_write(ring, mask);
 488         amdgpu_ring_write(ring, inv); /* poll interval */
 489 }
 490
 491 static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
 492 {
 493         /* Header itself is a NOP packet */
 494         if (num_nop == 1) {
 495                 amdgpu_ring_write(ring, ring->funcs->nop);
 496                 return;
 497         }
 498
 499         /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
 500         amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
 501
 502         /* Header is at index 0, followed by num_nops - 1 NOP packet's */
 503         amdgpu_ring_insert_nop(ring, num_nop - 1);
 504 }
 505
 506 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)
 507 {
 508         struct amdgpu_device *adev = ring->adev;
 509         uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 510         uint32_t tmp = 0;
 511         unsigned i;
 512         int r;
 513
 514         WREG32(scratch, 0xCAFEDEAD);
 515         r = amdgpu_ring_alloc(ring, 5);
 516         if (r) {
 517                 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
 518                           ring->idx, r);
 519                 return r;
 520         }
 521
 522         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
 523                 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
 524         } else {
 525                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 526                 amdgpu_ring_write(ring, scratch -
 527                                   PACKET3_SET_UCONFIG_REG_START);
 528                 amdgpu_ring_write(ring, 0xDEADBEEF);
 529         }
 530         amdgpu_ring_commit(ring);
 531
 532         for (i = 0; i < adev->usec_timeout; i++) {
 533                 tmp = RREG32(scratch);
 534                 if (tmp == 0xDEADBEEF)
 535                         break;
 536                 if (amdgpu_emu_mode == 1)
 537                         msleep(1);
 538                 else
 539                         udelay(1);
 540         }
 541
 542         if (i >= adev->usec_timeout)
 543                 r = -ETIMEDOUT;
 544         return r;
 545 }
 546
 547 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 548 {
 549         struct amdgpu_device *adev = ring->adev;
 550         struct amdgpu_ib ib;
 551         struct dma_fence *f = NULL;
 552         unsigned index;
 553         uint64_t gpu_addr;
 554         volatile uint32_t *cpu_ptr;
 555         long r;
 556
 557         /* MES KIQ fw hasn't indirect buffer support for now */
 558         if (adev->enable_mes_kiq &&
 559             ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 560                 return 0;
 561
 562         memset(&ib, 0, sizeof(ib));
 563
 564         if (ring->is_mes_queue) {
 565                 uint32_t padding, offset;
 566
 567                 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
 568                 padding = amdgpu_mes_ctx_get_offs(ring,
 569                                                   AMDGPU_MES_CTX_PADDING_OFFS);
 570
 571                 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 572                 ib.ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 573
 574                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, padding);
 575                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, padding);
 576                 *cpu_ptr = cpu_to_le32(0xCAFEDEAD);
 577         } else {
 578                 r = amdgpu_device_wb_get(adev, &index);
 579                 if (r)
 580                         return r;
 581
 582                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 583                 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 584                 cpu_ptr = &adev->wb.wb[index];
 585
 586                 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
 587                 if (r) {
 588                         DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 589                         goto err1;
 590                 }
 591         }
 592
 593         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
 594         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
 595         ib.ptr[2] = lower_32_bits(gpu_addr);
 596         ib.ptr[3] = upper_32_bits(gpu_addr);
 597         ib.ptr[4] = 0xDEADBEEF;
 598         ib.length_dw = 5;
 599
 600         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 601         if (r)
 602                 goto err2;
 603
 604         r = dma_fence_wait_timeout(f, false, timeout);
 605         if (r == 0) {
 606                 r = -ETIMEDOUT;
 607                 goto err2;
 608         } else if (r < 0) {
 609                 goto err2;
 610         }
 611
 612         if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
 613                 r = 0;
 614         else
 615                 r = -EINVAL;
 616 err2:
 617         if (!ring->is_mes_queue)
 618                 amdgpu_ib_free(adev, &ib, NULL);
 619         dma_fence_put(f);
 620 err1:
 621         if (!ring->is_mes_queue)
 622                 amdgpu_device_wb_free(adev, index);
 623         return r;
 624 }
 625
 626 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)
 627 {
 628         amdgpu_ucode_release(&adev->gfx.pfp_fw);
 629         amdgpu_ucode_release(&adev->gfx.me_fw);
 630         amdgpu_ucode_release(&adev->gfx.rlc_fw);
 631         amdgpu_ucode_release(&adev->gfx.mec_fw);
 632
 633         kfree(adev->gfx.rlc.register_list_format);
 634 }
 635
 636 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
 637 {
 638         const struct psp_firmware_header_v1_0 *toc_hdr;
 639         int err = 0;
 640
 641         err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
 642                                    "amdgpu/%s_toc.bin", ucode_prefix);
 643         if (err)
 644                 goto out;
 645
 646         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
 647         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
 648         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
 649         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
 650         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
 651                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
 652         return 0;
 653 out:
 654         amdgpu_ucode_release(&adev->psp.toc_fw);
 655         return err;
 656 }
 657
 658 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev)
 659 {
 660         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 661         case IP_VERSION(11, 0, 0):
 662         case IP_VERSION(11, 0, 2):
 663         case IP_VERSION(11, 0, 3):
 664                 if ((adev->gfx.me_fw_version >= 1505) &&
 665                     (adev->gfx.pfp_fw_version >= 1600) &&
 666                     (adev->gfx.mec_fw_version >= 512)) {
 667                         if (amdgpu_sriov_vf(adev))
 668                                 adev->gfx.cp_gfx_shadow = true;
 669                         else
 670                                 adev->gfx.cp_gfx_shadow = false;
 671                 }
 672                 break;
 673         default:
 674                 adev->gfx.cp_gfx_shadow = false;
 675                 break;
 676         }
 677 }
 678
 679 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 680 {
 681         char ucode_prefix[25];
 682         int err;
 683         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 684         uint16_t version_major;
 685         uint16_t version_minor;
 686
 687         DRM_DEBUG("\n");
 688
 689         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
 690         err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
 691                                    "amdgpu/%s_pfp.bin", ucode_prefix);
 692         if (err)
 693                 goto out;
 694         /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/
 695         adev->gfx.rs64_enable = amdgpu_ucode_hdr_version(
 696                                 (union amdgpu_firmware_header *)
 697                                 adev->gfx.pfp_fw->data, 2, 0);
 698         if (adev->gfx.rs64_enable) {
 699                 dev_info(adev->dev, "CP RS64 enable\n");
 700                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
 701                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
 702                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK);
 703         } else {
 704                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
 705         }
 706
 707         err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
 708                                    "amdgpu/%s_me.bin", ucode_prefix);
 709         if (err)
 710                 goto out;
 711         if (adev->gfx.rs64_enable) {
 712                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
 713                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
 714                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK);
 715         } else {
 716                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
 717         }
 718
 719         if (!amdgpu_sriov_vf(adev)) {
 720                 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) &&
 721                     adev->pdev->revision == 0xCE)
 722                         err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
 723                                                    "amdgpu/gc_11_0_0_rlc_1.bin");
 724                 else
 725                         err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
 726                                                    "amdgpu/%s_rlc.bin", ucode_prefix);
 727                 if (err)
 728                         goto out;
 729                 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 730                 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 731                 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
 732                 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
 733                 if (err)
 734                         goto out;
 735         }
 736
 737         err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
 738                                    "amdgpu/%s_mec.bin", ucode_prefix);
 739         if (err)
 740                 goto out;
 741         if (adev->gfx.rs64_enable) {
 742                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
 743                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
 744                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
 745                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
 746                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
 747         } else {
 748                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
 749                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
 750         }
 751
 752         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
 753                 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix);
 754
 755         /* only one MEC for gfx 11.0.0. */
 756         adev->gfx.mec2_fw = NULL;
 757
 758         gfx_v11_0_check_fw_cp_gfx_shadow(adev);
 759
 760         if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) {
 761                 err = adev->gfx.imu.funcs->init_microcode(adev);
 762                 if (err)
 763                         DRM_ERROR("Failed to init imu firmware!\n");
 764                 return err;
 765         }
 766
 767 out:
 768         if (err) {
 769                 amdgpu_ucode_release(&adev->gfx.pfp_fw);
 770                 amdgpu_ucode_release(&adev->gfx.me_fw);
 771                 amdgpu_ucode_release(&adev->gfx.rlc_fw);
 772                 amdgpu_ucode_release(&adev->gfx.mec_fw);
 773         }
 774
 775         return err;
 776 }
 777
 778 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev)
 779 {
 780         u32 count = 0;
 781         const struct cs_section_def *sect = NULL;
 782         const struct cs_extent_def *ext = NULL;
 783
 784         /* begin clear state */
 785         count += 2;
 786         /* context control state */
 787         count += 3;
 788
 789         for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
 790                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 791                         if (sect->id == SECT_CONTEXT)
 792                                 count += 2 + ext->reg_count;
 793                         else
 794                                 return 0;
 795                 }
 796         }
 797
 798         /* set PA_SC_TILE_STEERING_OVERRIDE */
 799         count += 3;
 800         /* end clear state */
 801         count += 2;
 802         /* clear state */
 803         count += 2;
 804
 805         return count;
 806 }
 807
 808 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev,
 809                                     volatile u32 *buffer)
 810 {
 811         u32 count = 0, i;
 812         const struct cs_section_def *sect = NULL;
 813         const struct cs_extent_def *ext = NULL;
 814         int ctx_reg_offset;
 815
 816         if (adev->gfx.rlc.cs_data == NULL)
 817                 return;
 818         if (buffer == NULL)
 819                 return;
 820
 821         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 822         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
 823
 824         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
 825         buffer[count++] = cpu_to_le32(0x80000000);
 826         buffer[count++] = cpu_to_le32(0x80000000);
 827
 828         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
 829                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 830                         if (sect->id == SECT_CONTEXT) {
 831                                 buffer[count++] =
 832                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
 833                                 buffer[count++] = cpu_to_le32(ext->reg_index -
 834                                                 PACKET3_SET_CONTEXT_REG_START);
 835                                 for (i = 0; i < ext->reg_count; i++)
 836                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
 837                         } else {
 838                                 return;
 839                         }
 840                 }
 841         }
 842
 843         ctx_reg_offset =
 844                 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
 845         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
 846         buffer[count++] = cpu_to_le32(ctx_reg_offset);
 847         buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
 848
 849         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 850         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
 851
 852         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
 853         buffer[count++] = cpu_to_le32(0);
 854 }
 855
 856 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev)
 857 {
 858         /* clear state block */
 859         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
 860                         &adev->gfx.rlc.clear_state_gpu_addr,
 861                         (void **)&adev->gfx.rlc.cs_ptr);
 862
 863         /* jump table block */
 864         amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
 865                         &adev->gfx.rlc.cp_table_gpu_addr,
 866                         (void **)&adev->gfx.rlc.cp_table_ptr);
 867 }
 868
 869 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
 870 {
 871         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
 872
 873         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
 874         reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 875         reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
 876         reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
 877         reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
 878         reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
 879         reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
 880         reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
 881         adev->gfx.rlc.rlcg_reg_access_supported = true;
 882 }
 883
 884 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
 885 {
 886         const struct cs_section_def *cs_data;
 887         int r;
 888
 889         adev->gfx.rlc.cs_data = gfx11_cs_data;
 890
 891         cs_data = adev->gfx.rlc.cs_data;
 892
 893         if (cs_data) {
 894                 /* init clear state block */
 895                 r = amdgpu_gfx_rlc_init_csb(adev);
 896                 if (r)
 897                         return r;
 898         }
 899
 900         /* init spm vmid with 0xf */
 901         if (adev->gfx.rlc.funcs->update_spm_vmid)
 902                 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
 903
 904         return 0;
 905 }
 906
 907 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev)
 908 {
 909         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
 910         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
 911         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
 912 }
 913
 914 static void gfx_v11_0_me_init(struct amdgpu_device *adev)
 915 {
 916         bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 917
 918         amdgpu_gfx_graphics_queue_acquire(adev);
 919 }
 920
 921 static int gfx_v11_0_mec_init(struct amdgpu_device *adev)
 922 {
 923         int r;
 924         u32 *hpd;
 925         size_t mec_hpd_size;
 926
 927         bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 928
 929         /* take ownership of the relevant compute queues */
 930         amdgpu_gfx_compute_queue_acquire(adev);
 931         mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE;
 932
 933         if (mec_hpd_size) {
 934                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
 935                                               AMDGPU_GEM_DOMAIN_GTT,
 936                                               &adev->gfx.mec.hpd_eop_obj,
 937                                               &adev->gfx.mec.hpd_eop_gpu_addr,
 938                                               (void **)&hpd);
 939                 if (r) {
 940                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
 941                         gfx_v11_0_mec_fini(adev);
 942                         return r;
 943                 }
 944
 945                 memset(hpd, 0, mec_hpd_size);
 946
 947                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
 948                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
 949         }
 950
 951         return 0;
 952 }
 953
 954 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
 955 {
 956         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 957                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 958                 (address << SQ_IND_INDEX__INDEX__SHIFT));
 959         return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 960 }
 961
 962 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
 963                            uint32_t thread, uint32_t regno,
 964                            uint32_t num, uint32_t *out)
 965 {
 966         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 967                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 968                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
 969                 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
 970                 (SQ_IND_INDEX__AUTO_INCR_MASK));
 971         while (num--)
 972                 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 973 }
 974
 975 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
 976 {
 977         /* in gfx11 the SIMD_ID is specified as part of the INSTANCE
 978          * field when performing a select_se_sh so it should be
 979          * zero here */
 980         WARN_ON(simd != 0);
 981
 982         /* type 3 wave data */
 983         dst[(*no_fields)++] = 3;
 984         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
 985         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
 986         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
 987         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
 988         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
 989         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
 990         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
 991         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
 992         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
 993         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
 994         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
 995         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
 996         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
 997         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
 998         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
 999 }
1000
1001 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1002                                      uint32_t wave, uint32_t start,
1003                                      uint32_t size, uint32_t *dst)
1004 {
1005         WARN_ON(simd != 0);
1006
1007         wave_read_regs(
1008                 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
1009                 dst);
1010 }
1011
1012 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1013                                       uint32_t wave, uint32_t thread,
1014                                       uint32_t start, uint32_t size,
1015                                       uint32_t *dst)
1016 {
1017         wave_read_regs(
1018                 adev, wave, thread,
1019                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1020 }
1021
1022 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,
1023                                         u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1024 {
1025         soc21_grbm_select(adev, me, pipe, q, vm);
1026 }
1027
1028 /* all sizes are in bytes */
1029 #define MQD_SHADOW_BASE_SIZE      73728
1030 #define MQD_SHADOW_BASE_ALIGNMENT 256
1031 #define MQD_FWWORKAREA_SIZE       484
1032 #define MQD_FWWORKAREA_ALIGNMENT  256
1033
1034 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev,
1035                                          struct amdgpu_gfx_shadow_info *shadow_info)
1036 {
1037         if (adev->gfx.cp_gfx_shadow) {
1038                 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE;
1039                 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT;
1040                 shadow_info->csa_size = MQD_FWWORKAREA_SIZE;
1041                 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT;
1042                 return 0;
1043         } else {
1044                 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info));
1045                 return -ENOTSUPP;
1046         }
1047 }
1048
1049 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
1050         .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter,
1051         .select_se_sh = &gfx_v11_0_select_se_sh,
1052         .read_wave_data = &gfx_v11_0_read_wave_data,
1053         .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs,
1054         .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs,
1055         .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
1056         .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk,
1057         .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info,
1058 };
1059
1060 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
1061 {
1062         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1063         case IP_VERSION(11, 0, 0):
1064         case IP_VERSION(11, 0, 2):
1065                 adev->gfx.config.max_hw_contexts = 8;
1066                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1067                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1068                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1069                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1070                 break;
1071         case IP_VERSION(11, 0, 3):
1072                 adev->gfx.ras = &gfx_v11_0_3_ras;
1073                 adev->gfx.config.max_hw_contexts = 8;
1074                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1075                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1076                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1077                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1078                 break;
1079         case IP_VERSION(11, 0, 1):
1080         case IP_VERSION(11, 0, 4):
1081         case IP_VERSION(11, 5, 0):
1082         case IP_VERSION(11, 5, 1):
1083         case IP_VERSION(11, 5, 2):
1084                 adev->gfx.config.max_hw_contexts = 8;
1085                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1086                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1087                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
1088                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300;
1089                 break;
1090         default:
1091                 BUG();
1092                 break;
1093         }
1094
1095         return 0;
1096 }
1097
1098 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
1099                                    int me, int pipe, int queue)
1100 {
1101         struct amdgpu_ring *ring;
1102         unsigned int irq_type;
1103         unsigned int hw_prio;
1104
1105         ring = &adev->gfx.gfx_ring[ring_id];
1106
1107         ring->me = me;
1108         ring->pipe = pipe;
1109         ring->queue = queue;
1110
1111         ring->ring_obj = NULL;
1112         ring->use_doorbell = true;
1113
1114         if (!ring_id)
1115                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
1116         else
1117                 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
1118         ring->vm_hub = AMDGPU_GFXHUB(0);
1119         sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1120
1121         irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
1122         hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ?
1123                 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1124         return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1125                                 hw_prio, NULL);
1126 }
1127
1128 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1129                                        int mec, int pipe, int queue)
1130 {
1131         int r;
1132         unsigned irq_type;
1133         struct amdgpu_ring *ring;
1134         unsigned int hw_prio;
1135
1136         ring = &adev->gfx.compute_ring[ring_id];
1137
1138         /* mec0 is me1 */
1139         ring->me = mec + 1;
1140         ring->pipe = pipe;
1141         ring->queue = queue;
1142
1143         ring->ring_obj = NULL;
1144         ring->use_doorbell = true;
1145         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1146         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1147                                 + (ring_id * GFX11_MEC_HPD_SIZE);
1148         ring->vm_hub = AMDGPU_GFXHUB(0);
1149         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1150
1151         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1152                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1153                 + ring->pipe;
1154         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1155                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1156         /* type-2 packets are deprecated on MEC, use type-3 instead */
1157         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1158                              hw_prio, NULL);
1159         if (r)
1160                 return r;
1161
1162         return 0;
1163 }
1164
1165 static struct {
1166         SOC21_FIRMWARE_ID       id;
1167         unsigned int            offset;
1168         unsigned int            size;
1169 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX];
1170
1171 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
1172 {
1173         RLC_TABLE_OF_CONTENT *ucode = rlc_toc;
1174
1175         while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) &&
1176                         (ucode->id < SOC21_FIRMWARE_ID_MAX)) {
1177                 rlc_autoload_info[ucode->id].id = ucode->id;
1178                 rlc_autoload_info[ucode->id].offset = ucode->offset * 4;
1179                 rlc_autoload_info[ucode->id].size = ucode->size * 4;
1180
1181                 ucode++;
1182         }
1183 }
1184
1185 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev)
1186 {
1187         uint32_t total_size = 0;
1188         SOC21_FIRMWARE_ID id;
1189
1190         gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
1191
1192         for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++)
1193                 total_size += rlc_autoload_info[id].size;
1194
1195         /* In case the offset in rlc toc ucode is aligned */
1196         if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset)
1197                 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset +
1198                         rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size;
1199
1200         return total_size;
1201 }
1202
1203 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
1204 {
1205         int r;
1206         uint32_t total_size;
1207
1208         total_size = gfx_v11_0_calc_toc_total_size(adev);
1209
1210         r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1211                                       AMDGPU_GEM_DOMAIN_VRAM |
1212                                       AMDGPU_GEM_DOMAIN_GTT,
1213                                       &adev->gfx.rlc.rlc_autoload_bo,
1214                                       &adev->gfx.rlc.rlc_autoload_gpu_addr,
1215                                       (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1216
1217         if (r) {
1218                 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1219                 return r;
1220         }
1221
1222         return 0;
1223 }
1224
1225 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1226                                               SOC21_FIRMWARE_ID id,
1227                                               const void *fw_data,
1228                                               uint32_t fw_size,
1229                                               uint32_t *fw_autoload_mask)
1230 {
1231         uint32_t toc_offset;
1232         uint32_t toc_fw_size;
1233         char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1234
1235         if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX)
1236                 return;
1237
1238         toc_offset = rlc_autoload_info[id].offset;
1239         toc_fw_size = rlc_autoload_info[id].size;
1240
1241         if (fw_size == 0)
1242                 fw_size = toc_fw_size;
1243
1244         if (fw_size > toc_fw_size)
1245                 fw_size = toc_fw_size;
1246
1247         memcpy(ptr + toc_offset, fw_data, fw_size);
1248
1249         if (fw_size < toc_fw_size)
1250                 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
1251
1252         if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME))
1253                 *(uint64_t *)fw_autoload_mask |= 1ULL << id;
1254 }
1255
1256 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev,
1257                                                         uint32_t *fw_autoload_mask)
1258 {
1259         void *data;
1260         uint32_t size;
1261         uint64_t *toc_ptr;
1262
1263         *(uint64_t *)fw_autoload_mask |= 0x1;
1264
1265         DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask);
1266
1267         data = adev->psp.toc.start_addr;
1268         size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size;
1269
1270         toc_ptr = (uint64_t *)data + size / 8 - 1;
1271         *toc_ptr = *(uint64_t *)fw_autoload_mask;
1272
1273         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC,
1274                                         data, size, fw_autoload_mask);
1275 }
1276
1277 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev,
1278                                                         uint32_t *fw_autoload_mask)
1279 {
1280         const __le32 *fw_data;
1281         uint32_t fw_size;
1282         const struct gfx_firmware_header_v1_0 *cp_hdr;
1283         const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1284         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1285         const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1286         uint16_t version_major, version_minor;
1287
1288         if (adev->gfx.rs64_enable) {
1289                 /* pfp ucode */
1290                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1291                         adev->gfx.pfp_fw->data;
1292                 /* instruction */
1293                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1294                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1295                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1296                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP,
1297                                                 fw_data, fw_size, fw_autoload_mask);
1298                 /* data */
1299                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1300                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1301                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1302                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK,
1303                                                 fw_data, fw_size, fw_autoload_mask);
1304                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK,
1305                                                 fw_data, fw_size, fw_autoload_mask);
1306                 /* me ucode */
1307                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1308                         adev->gfx.me_fw->data;
1309                 /* instruction */
1310                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1311                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1312                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1313                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME,
1314                                                 fw_data, fw_size, fw_autoload_mask);
1315                 /* data */
1316                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1317                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1318                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1319                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK,
1320                                                 fw_data, fw_size, fw_autoload_mask);
1321                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK,
1322                                                 fw_data, fw_size, fw_autoload_mask);
1323                 /* mec ucode */
1324                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1325                         adev->gfx.mec_fw->data;
1326                 /* instruction */
1327                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1328                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1329                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1330                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC,
1331                                                 fw_data, fw_size, fw_autoload_mask);
1332                 /* data */
1333                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1334                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1335                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1336                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK,
1337                                                 fw_data, fw_size, fw_autoload_mask);
1338                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK,
1339                                                 fw_data, fw_size, fw_autoload_mask);
1340                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK,
1341                                                 fw_data, fw_size, fw_autoload_mask);
1342                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK,
1343                                                 fw_data, fw_size, fw_autoload_mask);
1344         } else {
1345                 /* pfp ucode */
1346                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1347                         adev->gfx.pfp_fw->data;
1348                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1349                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1350                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1351                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP,
1352                                                 fw_data, fw_size, fw_autoload_mask);
1353
1354                 /* me ucode */
1355                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1356                         adev->gfx.me_fw->data;
1357                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1358                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1359                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1360                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME,
1361                                                 fw_data, fw_size, fw_autoload_mask);
1362
1363                 /* mec ucode */
1364                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1365                         adev->gfx.mec_fw->data;
1366                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1367                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1368                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1369                         cp_hdr->jt_size * 4;
1370                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC,
1371                                                 fw_data, fw_size, fw_autoload_mask);
1372         }
1373
1374         /* rlc ucode */
1375         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1376                 adev->gfx.rlc_fw->data;
1377         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1378                         le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1379         fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1380         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE,
1381                                         fw_data, fw_size, fw_autoload_mask);
1382
1383         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1384         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1385         if (version_major == 2) {
1386                 if (version_minor >= 2) {
1387                         rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1388
1389                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1390                                         le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1391                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1392                         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE,
1393                                         fw_data, fw_size, fw_autoload_mask);
1394
1395                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1396                                         le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1397                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1398                         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT,
1399                                         fw_data, fw_size, fw_autoload_mask);
1400                 }
1401         }
1402 }
1403
1404 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev,
1405                                                         uint32_t *fw_autoload_mask)
1406 {
1407         const __le32 *fw_data;
1408         uint32_t fw_size;
1409         const struct sdma_firmware_header_v2_0 *sdma_hdr;
1410
1411         sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
1412                 adev->sdma.instance[0].fw->data;
1413         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1414                         le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
1415         fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes);
1416
1417         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1418                         SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask);
1419
1420         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1421                         le32_to_cpu(sdma_hdr->ctl_ucode_offset));
1422         fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes);
1423
1424         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1425                         SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask);
1426 }
1427
1428 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev,
1429                                                         uint32_t *fw_autoload_mask)
1430 {
1431         const __le32 *fw_data;
1432         unsigned fw_size;
1433         const struct mes_firmware_header_v1_0 *mes_hdr;
1434         int pipe, ucode_id, data_id;
1435
1436         for (pipe = 0; pipe < 2; pipe++) {
1437                 if (pipe==0) {
1438                         ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0;
1439                         data_id  = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK;
1440                 } else {
1441                         ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1;
1442                         data_id  = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK;
1443                 }
1444
1445                 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1446                         adev->mes.fw[pipe]->data;
1447
1448                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1449                                 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1450                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1451
1452                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1453                                 ucode_id, fw_data, fw_size, fw_autoload_mask);
1454
1455                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1456                                 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1457                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1458
1459                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1460                                 data_id, fw_data, fw_size, fw_autoload_mask);
1461         }
1462 }
1463
1464 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1465 {
1466         uint32_t rlc_g_offset, rlc_g_size;
1467         uint64_t gpu_addr;
1468         uint32_t autoload_fw_id[2];
1469
1470         memset(autoload_fw_id, 0, sizeof(uint32_t) * 2);
1471
1472         /* RLC autoload sequence 2: copy ucode */
1473         gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id);
1474         gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id);
1475         gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id);
1476         gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id);
1477
1478         rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset;
1479         rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size;
1480         gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
1481
1482         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1483         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1484
1485         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1486
1487         /* RLC autoload sequence 3: load IMU fw */
1488         if (adev->gfx.imu.funcs->load_microcode)
1489                 adev->gfx.imu.funcs->load_microcode(adev);
1490         /* RLC autoload sequence 4 init IMU fw */
1491         if (adev->gfx.imu.funcs->setup_imu)
1492                 adev->gfx.imu.funcs->setup_imu(adev);
1493         if (adev->gfx.imu.funcs->start_imu)
1494                 adev->gfx.imu.funcs->start_imu(adev);
1495
1496         /* RLC autoload sequence 5 disable gpa mode */
1497         gfx_v11_0_disable_gpa_mode(adev);
1498
1499         return 0;
1500 }
1501
1502 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev)
1503 {
1504         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
1505         uint32_t *ptr;
1506         uint32_t inst;
1507
1508         ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
1509         if (!ptr) {
1510                 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
1511                 adev->gfx.ip_dump_core = NULL;
1512         } else {
1513                 adev->gfx.ip_dump_core = ptr;
1514         }
1515
1516         /* Allocate memory for compute queue registers for all the instances */
1517         reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
1518         inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1519                 adev->gfx.mec.num_queue_per_pipe;
1520
1521         ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1522         if (!ptr) {
1523                 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
1524                 adev->gfx.ip_dump_compute_queues = NULL;
1525         } else {
1526                 adev->gfx.ip_dump_compute_queues = ptr;
1527         }
1528
1529         /* Allocate memory for gfx queue registers for all the instances */
1530         reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
1531         inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me *
1532                 adev->gfx.me.num_queue_per_pipe;
1533
1534         ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1535         if (!ptr) {
1536                 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n");
1537                 adev->gfx.ip_dump_gfx_queues = NULL;
1538         } else {
1539                 adev->gfx.ip_dump_gfx_queues = ptr;
1540         }
1541 }
1542
1543 static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
1544 {
1545         int i, j, k, r, ring_id = 0;
1546         int xcc_id = 0;
1547         struct amdgpu_device *adev = ip_block->adev;
1548
1549         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1550         case IP_VERSION(11, 0, 0):
1551         case IP_VERSION(11, 0, 2):
1552         case IP_VERSION(11, 0, 3):
1553                 adev->gfx.me.num_me = 1;
1554                 adev->gfx.me.num_pipe_per_me = 1;
1555                 adev->gfx.me.num_queue_per_pipe = 1;
1556                 adev->gfx.mec.num_mec = 2;
1557                 adev->gfx.mec.num_pipe_per_mec = 4;
1558                 adev->gfx.mec.num_queue_per_pipe = 4;
1559                 break;
1560         case IP_VERSION(11, 0, 1):
1561         case IP_VERSION(11, 0, 4):
1562         case IP_VERSION(11, 5, 0):
1563         case IP_VERSION(11, 5, 1):
1564         case IP_VERSION(11, 5, 2):
1565                 adev->gfx.me.num_me = 1;
1566                 adev->gfx.me.num_pipe_per_me = 1;
1567                 adev->gfx.me.num_queue_per_pipe = 1;
1568                 adev->gfx.mec.num_mec = 1;
1569                 adev->gfx.mec.num_pipe_per_mec = 4;
1570                 adev->gfx.mec.num_queue_per_pipe = 4;
1571                 break;
1572         default:
1573                 adev->gfx.me.num_me = 1;
1574                 adev->gfx.me.num_pipe_per_me = 1;
1575                 adev->gfx.me.num_queue_per_pipe = 1;
1576                 adev->gfx.mec.num_mec = 1;
1577                 adev->gfx.mec.num_pipe_per_mec = 4;
1578                 adev->gfx.mec.num_queue_per_pipe = 8;
1579                 break;
1580         }
1581
1582         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1583         case IP_VERSION(11, 0, 3):
1584                 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1585                 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1586                 if (adev->gfx.me_fw_version  >= 2280 &&
1587                     adev->gfx.pfp_fw_version >= 2370 &&
1588                     adev->gfx.mec_fw_version >= 2450  &&
1589                     adev->mes.fw_version[0] >= 99) {
1590                         adev->gfx.enable_cleaner_shader = true;
1591                         r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1592                         if (r) {
1593                                 adev->gfx.enable_cleaner_shader = false;
1594                                 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1595                         }
1596                 }
1597                 break;
1598         default:
1599                 adev->gfx.enable_cleaner_shader = false;
1600                 break;
1601         }
1602
1603         /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */
1604         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) &&
1605             amdgpu_sriov_is_pp_one_vf(adev))
1606                 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG;
1607
1608         /* EOP Event */
1609         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1610                               GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1611                               &adev->gfx.eop_irq);
1612         if (r)
1613                 return r;
1614
1615         /* Bad opcode Event */
1616         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1617                               GFX_11_0_0__SRCID__CP_BAD_OPCODE_ERROR,
1618                               &adev->gfx.bad_op_irq);
1619         if (r)
1620                 return r;
1621
1622         /* Privileged reg */
1623         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1624                               GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1625                               &adev->gfx.priv_reg_irq);
1626         if (r)
1627                 return r;
1628
1629         /* Privileged inst */
1630         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1631                               GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1632                               &adev->gfx.priv_inst_irq);
1633         if (r)
1634                 return r;
1635
1636         /* FED error */
1637         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1638                                   GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT,
1639                                   &adev->gfx.rlc_gc_fed_irq);
1640         if (r)
1641                 return r;
1642
1643         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1644
1645         gfx_v11_0_me_init(adev);
1646
1647         r = gfx_v11_0_rlc_init(adev);
1648         if (r) {
1649                 DRM_ERROR("Failed to init rlc BOs!\n");
1650                 return r;
1651         }
1652
1653         r = gfx_v11_0_mec_init(adev);
1654         if (r) {
1655                 DRM_ERROR("Failed to init MEC BOs!\n");
1656                 return r;
1657         }
1658
1659         /* set up the gfx ring */
1660         for (i = 0; i < adev->gfx.me.num_me; i++) {
1661                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1662                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1663                                 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1664                                         continue;
1665
1666                                 r = gfx_v11_0_gfx_ring_init(adev, ring_id,
1667                                                             i, k, j);
1668                                 if (r)
1669                                         return r;
1670                                 ring_id++;
1671                         }
1672                 }
1673         }
1674
1675         ring_id = 0;
1676         /* set up the compute queues - allocate horizontally across pipes */
1677         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1678                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1679                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1680                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
1681                                                                      k, j))
1682                                         continue;
1683
1684                                 r = gfx_v11_0_compute_ring_init(adev, ring_id,
1685                                                                 i, k, j);
1686                                 if (r)
1687                                         return r;
1688
1689                                 ring_id++;
1690                         }
1691                 }
1692         }
1693
1694         if (!adev->enable_mes_kiq) {
1695                 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0);
1696                 if (r) {
1697                         DRM_ERROR("Failed to init KIQ BOs!\n");
1698                         return r;
1699                 }
1700
1701                 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1702                 if (r)
1703                         return r;
1704         }
1705
1706         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0);
1707         if (r)
1708                 return r;
1709
1710         /* allocate visible FB for rlc auto-loading fw */
1711         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1712                 r = gfx_v11_0_rlc_autoload_buffer_init(adev);
1713                 if (r)
1714                         return r;
1715         }
1716
1717         r = gfx_v11_0_gpu_early_init(adev);
1718         if (r)
1719                 return r;
1720
1721         if (amdgpu_gfx_ras_sw_init(adev)) {
1722                 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
1723                 return -EINVAL;
1724         }
1725
1726         gfx_v11_0_alloc_ip_dump(adev);
1727
1728         r = amdgpu_gfx_sysfs_init(adev);
1729         if (r)
1730                 return r;
1731
1732         return 0;
1733 }
1734
1735 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev)
1736 {
1737         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1738                               &adev->gfx.pfp.pfp_fw_gpu_addr,
1739                               (void **)&adev->gfx.pfp.pfp_fw_ptr);
1740
1741         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1742                               &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1743                               (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1744 }
1745
1746 static void gfx_v11_0_me_fini(struct amdgpu_device *adev)
1747 {
1748         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1749                               &adev->gfx.me.me_fw_gpu_addr,
1750                               (void **)&adev->gfx.me.me_fw_ptr);
1751
1752         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1753                                &adev->gfx.me.me_fw_data_gpu_addr,
1754                                (void **)&adev->gfx.me.me_fw_data_ptr);
1755 }
1756
1757 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1758 {
1759         amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1760                         &adev->gfx.rlc.rlc_autoload_gpu_addr,
1761                         (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1762 }
1763
1764 static int gfx_v11_0_sw_fini(struct amdgpu_ip_block *ip_block)
1765 {
1766         int i;
1767         struct amdgpu_device *adev = ip_block->adev;
1768
1769         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1770                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1771         for (i = 0; i < adev->gfx.num_compute_rings; i++)
1772                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1773
1774         amdgpu_gfx_mqd_sw_fini(adev, 0);
1775
1776         if (!adev->enable_mes_kiq) {
1777                 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
1778                 amdgpu_gfx_kiq_fini(adev, 0);
1779         }
1780
1781         amdgpu_gfx_cleaner_shader_sw_fini(adev);
1782
1783         gfx_v11_0_pfp_fini(adev);
1784         gfx_v11_0_me_fini(adev);
1785         gfx_v11_0_rlc_fini(adev);
1786         gfx_v11_0_mec_fini(adev);
1787
1788         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1789                 gfx_v11_0_rlc_autoload_buffer_fini(adev);
1790
1791         gfx_v11_0_free_microcode(adev);
1792
1793         amdgpu_gfx_sysfs_fini(adev);
1794
1795         kfree(adev->gfx.ip_dump_core);
1796         kfree(adev->gfx.ip_dump_compute_queues);
1797         kfree(adev->gfx.ip_dump_gfx_queues);
1798
1799         return 0;
1800 }
1801
1802 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1803                                    u32 sh_num, u32 instance, int xcc_id)
1804 {
1805         u32 data;
1806
1807         if (instance == 0xffffffff)
1808                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1809                                      INSTANCE_BROADCAST_WRITES, 1);
1810         else
1811                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1812                                      instance);
1813
1814         if (se_num == 0xffffffff)
1815                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1816                                      1);
1817         else
1818                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1819
1820         if (sh_num == 0xffffffff)
1821                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1822                                      1);
1823         else
1824                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1825
1826         WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
1827 }
1828
1829 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev)
1830 {
1831         u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1832
1833         gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE);
1834         gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1835                                            CC_GC_SA_UNIT_DISABLE,
1836                                            SA_DISABLE);
1837         gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE);
1838         gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1839                                                  GC_USER_SA_UNIT_DISABLE,
1840                                                  SA_DISABLE);
1841         sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1842                                             adev->gfx.config.max_shader_engines);
1843
1844         return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1845 }
1846
1847 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1848 {
1849         u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1850         u32 rb_mask;
1851
1852         gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
1853         gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1854                                             CC_RB_BACKEND_DISABLE,
1855                                             BACKEND_DISABLE);
1856         gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
1857         gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
1858                                                  GC_USER_RB_BACKEND_DISABLE,
1859                                                  BACKEND_DISABLE);
1860         rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1861                                             adev->gfx.config.max_shader_engines);
1862
1863         return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
1864 }
1865
1866 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev)
1867 {
1868         u32 rb_bitmap_width_per_sa;
1869         u32 max_sa;
1870         u32 active_sa_bitmap;
1871         u32 global_active_rb_bitmap;
1872         u32 active_rb_bitmap = 0;
1873         u32 i;
1874
1875         /* query sa bitmap from SA_UNIT_DISABLE registers */
1876         active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev);
1877         /* query rb bitmap from RB_BACKEND_DISABLE registers */
1878         global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev);
1879
1880         /* generate active rb bitmap according to active sa bitmap */
1881         max_sa = adev->gfx.config.max_shader_engines *
1882                  adev->gfx.config.max_sh_per_se;
1883         rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1884                                  adev->gfx.config.max_sh_per_se;
1885         for (i = 0; i < max_sa; i++) {
1886                 if (active_sa_bitmap & (1 << i))
1887                         active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
1888         }
1889
1890         active_rb_bitmap &= global_active_rb_bitmap;
1891         adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1892         adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1893 }
1894
1895 #define DEFAULT_SH_MEM_BASES    (0x6000)
1896 #define LDS_APP_BASE           0x1
1897 #define SCRATCH_APP_BASE       0x2
1898
1899 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
1900 {
1901         int i;
1902         uint32_t sh_mem_bases;
1903         uint32_t data;
1904
1905         /*
1906          * Configure apertures:
1907          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1908          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1909          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1910          */
1911         sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
1912                         SCRATCH_APP_BASE;
1913
1914         mutex_lock(&adev->srbm_mutex);
1915         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1916                 soc21_grbm_select(adev, 0, 0, 0, i);
1917                 /* CP and shaders */
1918                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1919                 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
1920
1921                 /* Enable trap for each kfd vmid. */
1922                 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL);
1923                 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1924                 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data);
1925         }
1926         soc21_grbm_select(adev, 0, 0, 0, 0);
1927         mutex_unlock(&adev->srbm_mutex);
1928
1929         /*
1930          * Initialize all compute VMIDs to have no GDS, GWS, or OA
1931          * access. These should be enabled by FW for target VMIDs.
1932          */
1933         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1934                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0);
1935                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0);
1936                 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0);
1937                 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0);
1938         }
1939 }
1940
1941 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev)
1942 {
1943         int vmid;
1944
1945         /*
1946          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
1947          * access. Compute VMIDs should be enabled by FW for target VMIDs,
1948          * the driver can enable them for graphics. VMID0 should maintain
1949          * access so that HWS firmware can save/restore entries.
1950          */
1951         for (vmid = 1; vmid < 16; vmid++) {
1952                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0);
1953                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0);
1954                 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0);
1955                 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0);
1956         }
1957 }
1958
1959 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev)
1960 {
1961         /* TODO: harvest feature to be added later. */
1962 }
1963
1964 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev)
1965 {
1966         /* TCCs are global (not instanced). */
1967         uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) |
1968                                RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE);
1969
1970         adev->gfx.config.tcc_disabled_mask =
1971                 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) |
1972                 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16);
1973 }
1974
1975 static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
1976 {
1977         u32 tmp;
1978         int i;
1979
1980         if (!amdgpu_sriov_vf(adev))
1981                 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1982
1983         gfx_v11_0_setup_rb(adev);
1984         gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info);
1985         gfx_v11_0_get_tcc_info(adev);
1986         adev->gfx.config.pa_sc_tile_steering_override = 0;
1987
1988         /* Set whether texture coordinate truncation is conformant. */
1989         tmp = RREG32_SOC15(GC, 0, regTA_CNTL2);
1990         adev->gfx.config.ta_cntl2_truncate_coord_mode =
1991                 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE);
1992
1993         /* XXX SH_MEM regs */
1994         /* where to put LDS, scratch, GPUVM in FSA64 space */
1995         mutex_lock(&adev->srbm_mutex);
1996         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1997                 soc21_grbm_select(adev, 0, 0, 0, i);
1998                 /* CP and shaders */
1999                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
2000                 if (i != 0) {
2001                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2002                                 (adev->gmc.private_aperture_start >> 48));
2003                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2004                                 (adev->gmc.shared_aperture_start >> 48));
2005                         WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
2006                 }
2007         }
2008         soc21_grbm_select(adev, 0, 0, 0, 0);
2009
2010         mutex_unlock(&adev->srbm_mutex);
2011
2012         gfx_v11_0_init_compute_vmid(adev);
2013         gfx_v11_0_init_gds_vmid(adev);
2014 }
2015
2016 static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev,
2017                                       int me, int pipe)
2018 {
2019         if (me != 0)
2020                 return 0;
2021
2022         switch (pipe) {
2023         case 0:
2024                 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
2025         case 1:
2026                 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
2027         default:
2028                 return 0;
2029         }
2030 }
2031
2032 static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev,
2033                                       int me, int pipe)
2034 {
2035         /*
2036          * amdgpu controls only the first MEC. That's why this function only
2037          * handles the setting of interrupts for this specific MEC. All other
2038          * pipes' interrupts are set by amdkfd.
2039          */
2040         if (me != 1)
2041                 return 0;
2042
2043         switch (pipe) {
2044         case 0:
2045                 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
2046         case 1:
2047                 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
2048         case 2:
2049                 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
2050         case 3:
2051                 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
2052         default:
2053                 return 0;
2054         }
2055 }
2056
2057 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2058                                                bool enable)
2059 {
2060         u32 tmp, cp_int_cntl_reg;
2061         int i, j;
2062
2063         if (amdgpu_sriov_vf(adev))
2064                 return;
2065
2066         for (i = 0; i < adev->gfx.me.num_me; i++) {
2067                 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
2068                         cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
2069
2070                         if (cp_int_cntl_reg) {
2071                                 tmp = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
2072                                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
2073                                                     enable ? 1 : 0);
2074                                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
2075                                                     enable ? 1 : 0);
2076                                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
2077                                                     enable ? 1 : 0);
2078                                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
2079                                                     enable ? 1 : 0);
2080                                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, tmp);
2081                         }
2082                 }
2083         }
2084 }
2085
2086 static int gfx_v11_0_init_csb(struct amdgpu_device *adev)
2087 {
2088         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2089
2090         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
2091                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
2092         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
2093                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2094         WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
2095
2096         return 0;
2097 }
2098
2099 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev)
2100 {
2101         u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
2102
2103         tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
2104         WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
2105 }
2106
2107 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev)
2108 {
2109         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2110         udelay(50);
2111         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2112         udelay(50);
2113 }
2114
2115 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
2116                                              bool enable)
2117 {
2118         uint32_t rlc_pg_cntl;
2119
2120         rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
2121
2122         if (!enable) {
2123                 /* RLC_PG_CNTL[23] = 0 (default)
2124                  * RLC will wait for handshake acks with SMU
2125                  * GFXOFF will be enabled
2126                  * RLC_PG_CNTL[23] = 1
2127                  * RLC will not issue any message to SMU
2128                  * hence no handshake between SMU & RLC
2129                  * GFXOFF will be disabled
2130                  */
2131                 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2132         } else
2133                 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2134         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
2135 }
2136
2137 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev)
2138 {
2139         /* TODO: enable rlc & smu handshake until smu
2140          * and gfxoff feature works as expected */
2141         if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
2142                 gfx_v11_0_rlc_smu_handshake_cntl(adev, false);
2143
2144         WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2145         udelay(50);
2146 }
2147
2148 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev)
2149 {
2150         uint32_t tmp;
2151
2152         /* enable Save Restore Machine */
2153         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
2154         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2155         tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
2156         WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
2157 }
2158
2159 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev)
2160 {
2161         const struct rlc_firmware_header_v2_0 *hdr;
2162         const __le32 *fw_data;
2163         unsigned i, fw_size;
2164
2165         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2166         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2167                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2168         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2169
2170         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
2171                      RLCG_UCODE_LOADING_START_ADDRESS);
2172
2173         for (i = 0; i < fw_size; i++)
2174                 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
2175                              le32_to_cpup(fw_data++));
2176
2177         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2178 }
2179
2180 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
2181 {
2182         const struct rlc_firmware_header_v2_2 *hdr;
2183         const __le32 *fw_data;
2184         unsigned i, fw_size;
2185         u32 tmp;
2186
2187         hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
2188
2189         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2190                         le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
2191         fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
2192
2193         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
2194
2195         for (i = 0; i < fw_size; i++) {
2196                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2197                         msleep(1);
2198                 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
2199                                 le32_to_cpup(fw_data++));
2200         }
2201
2202         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2203
2204         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2205                         le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
2206         fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
2207
2208         WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
2209         for (i = 0; i < fw_size; i++) {
2210                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2211                         msleep(1);
2212                 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
2213                                 le32_to_cpup(fw_data++));
2214         }
2215
2216         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2217
2218         tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
2219         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
2220         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
2221         WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
2222 }
2223
2224 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev)
2225 {
2226         const struct rlc_firmware_header_v2_3 *hdr;
2227         const __le32 *fw_data;
2228         unsigned i, fw_size;
2229         u32 tmp;
2230
2231         hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
2232
2233         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2234                         le32_to_cpu(hdr->rlcp_ucode_offset_bytes));
2235         fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4;
2236
2237         WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0);
2238
2239         for (i = 0; i < fw_size; i++) {
2240                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2241                         msleep(1);
2242                 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA,
2243                                 le32_to_cpup(fw_data++));
2244         }
2245
2246         WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version);
2247
2248         tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
2249         tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
2250         WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp);
2251
2252         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2253                         le32_to_cpu(hdr->rlcv_ucode_offset_bytes));
2254         fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4;
2255
2256         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0);
2257
2258         for (i = 0; i < fw_size; i++) {
2259                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2260                         msleep(1);
2261                 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA,
2262                                 le32_to_cpup(fw_data++));
2263         }
2264
2265         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version);
2266
2267         tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL);
2268         tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1);
2269         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp);
2270 }
2271
2272 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev)
2273 {
2274         const struct rlc_firmware_header_v2_0 *hdr;
2275         uint16_t version_major;
2276         uint16_t version_minor;
2277
2278         if (!adev->gfx.rlc_fw)
2279                 return -EINVAL;
2280
2281         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2282         amdgpu_ucode_print_rlc_hdr(&hdr->header);
2283
2284         version_major = le16_to_cpu(hdr->header.header_version_major);
2285         version_minor = le16_to_cpu(hdr->header.header_version_minor);
2286
2287         if (version_major == 2) {
2288                 gfx_v11_0_load_rlcg_microcode(adev);
2289                 if (amdgpu_dpm == 1) {
2290                         if (version_minor >= 2)
2291                                 gfx_v11_0_load_rlc_iram_dram_microcode(adev);
2292                         if (version_minor == 3)
2293                                 gfx_v11_0_load_rlcp_rlcv_microcode(adev);
2294                 }
2295
2296                 return 0;
2297         }
2298
2299         return -EINVAL;
2300 }
2301
2302 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev)
2303 {
2304         int r;
2305
2306         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
2307                 gfx_v11_0_init_csb(adev);
2308
2309                 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
2310                         gfx_v11_0_rlc_enable_srm(adev);
2311         } else {
2312                 if (amdgpu_sriov_vf(adev)) {
2313                         gfx_v11_0_init_csb(adev);
2314                         return 0;
2315                 }
2316
2317                 adev->gfx.rlc.funcs->stop(adev);
2318
2319                 /* disable CG */
2320                 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
2321
2322                 /* disable PG */
2323                 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
2324
2325                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2326                         /* legacy rlc firmware loading */
2327                         r = gfx_v11_0_rlc_load_microcode(adev);
2328                         if (r)
2329                                 return r;
2330                 }
2331
2332                 gfx_v11_0_init_csb(adev);
2333
2334                 adev->gfx.rlc.funcs->start(adev);
2335         }
2336         return 0;
2337 }
2338
2339 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr)
2340 {
2341         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2342         uint32_t tmp;
2343         int i;
2344
2345         /* Trigger an invalidation of the L1 instruction caches */
2346         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2347         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2348         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2349
2350         /* Wait for invalidation complete */
2351         for (i = 0; i < usec_timeout; i++) {
2352                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2353                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2354                                         INVALIDATE_CACHE_COMPLETE))
2355                         break;
2356                 udelay(1);
2357         }
2358
2359         if (i >= usec_timeout) {
2360                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2361                 return -EINVAL;
2362         }
2363
2364         if (amdgpu_emu_mode == 1)
2365                 adev->hdp.funcs->flush_hdp(adev, NULL);
2366
2367         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2368         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2369         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2370         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2371         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2372         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2373
2374         /* Program me ucode address into intruction cache address register */
2375         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2376                         lower_32_bits(addr) & 0xFFFFF000);
2377         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2378                         upper_32_bits(addr));
2379
2380         return 0;
2381 }
2382
2383 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr)
2384 {
2385         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2386         uint32_t tmp;
2387         int i;
2388
2389         /* Trigger an invalidation of the L1 instruction caches */
2390         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2391         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2392         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2393
2394         /* Wait for invalidation complete */
2395         for (i = 0; i < usec_timeout; i++) {
2396                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2397                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2398                                         INVALIDATE_CACHE_COMPLETE))
2399                         break;
2400                 udelay(1);
2401         }
2402
2403         if (i >= usec_timeout) {
2404                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2405                 return -EINVAL;
2406         }
2407
2408         if (amdgpu_emu_mode == 1)
2409                 adev->hdp.funcs->flush_hdp(adev, NULL);
2410
2411         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2412         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2413         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2414         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2415         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2416         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2417
2418         /* Program pfp ucode address into intruction cache address register */
2419         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2420                         lower_32_bits(addr) & 0xFFFFF000);
2421         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2422                         upper_32_bits(addr));
2423
2424         return 0;
2425 }
2426
2427 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr)
2428 {
2429         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2430         uint32_t tmp;
2431         int i;
2432
2433         /* Trigger an invalidation of the L1 instruction caches */
2434         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2435         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2436
2437         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2438
2439         /* Wait for invalidation complete */
2440         for (i = 0; i < usec_timeout; i++) {
2441                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2442                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2443                                         INVALIDATE_CACHE_COMPLETE))
2444                         break;
2445                 udelay(1);
2446         }
2447
2448         if (i >= usec_timeout) {
2449                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2450                 return -EINVAL;
2451         }
2452
2453         if (amdgpu_emu_mode == 1)
2454                 adev->hdp.funcs->flush_hdp(adev, NULL);
2455
2456         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2457         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2458         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2459         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2460         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2461
2462         /* Program mec1 ucode address into intruction cache address register */
2463         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2464                         lower_32_bits(addr) & 0xFFFFF000);
2465         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2466                         upper_32_bits(addr));
2467
2468         return 0;
2469 }
2470
2471 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2472 {
2473         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2474         uint32_t tmp;
2475         unsigned i, pipe_id;
2476         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2477
2478         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2479                 adev->gfx.pfp_fw->data;
2480
2481         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2482                 lower_32_bits(addr));
2483         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2484                 upper_32_bits(addr));
2485
2486         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2487         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2488         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2489         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2490         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2491
2492         /*
2493          * Programming any of the CP_PFP_IC_BASE registers
2494          * forces invalidation of the ME L1 I$. Wait for the
2495          * invalidation complete
2496          */
2497         for (i = 0; i < usec_timeout; i++) {
2498                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2499                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2500                         INVALIDATE_CACHE_COMPLETE))
2501                         break;
2502                 udelay(1);
2503         }
2504
2505         if (i >= usec_timeout) {
2506                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2507                 return -EINVAL;
2508         }
2509
2510         /* Prime the L1 instruction caches */
2511         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2512         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2513         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2514         /* Waiting for cache primed*/
2515         for (i = 0; i < usec_timeout; i++) {
2516                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2517                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2518                         ICACHE_PRIMED))
2519                         break;
2520                 udelay(1);
2521         }
2522
2523         if (i >= usec_timeout) {
2524                 dev_err(adev->dev, "failed to prime instruction cache\n");
2525                 return -EINVAL;
2526         }
2527
2528         mutex_lock(&adev->srbm_mutex);
2529         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2530                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2531                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2532                         (pfp_hdr->ucode_start_addr_hi << 30) |
2533                         (pfp_hdr->ucode_start_addr_lo >> 2));
2534                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2535                         pfp_hdr->ucode_start_addr_hi >> 2);
2536
2537                 /*
2538                  * Program CP_ME_CNTL to reset given PIPE to take
2539                  * effect of CP_PFP_PRGRM_CNTR_START.
2540                  */
2541                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2542                 if (pipe_id == 0)
2543                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2544                                         PFP_PIPE0_RESET, 1);
2545                 else
2546                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2547                                         PFP_PIPE1_RESET, 1);
2548                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2549
2550                 /* Clear pfp pipe0 reset bit. */
2551                 if (pipe_id == 0)
2552                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2553                                         PFP_PIPE0_RESET, 0);
2554                 else
2555                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2556                                         PFP_PIPE1_RESET, 0);
2557                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2558
2559                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2560                         lower_32_bits(addr2));
2561                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2562                         upper_32_bits(addr2));
2563         }
2564         soc21_grbm_select(adev, 0, 0, 0, 0);
2565         mutex_unlock(&adev->srbm_mutex);
2566
2567         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2568         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2569         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2570         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2571
2572         /* Invalidate the data caches */
2573         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2574         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2575         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2576
2577         for (i = 0; i < usec_timeout; i++) {
2578                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2579                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2580                         INVALIDATE_DCACHE_COMPLETE))
2581                         break;
2582                 udelay(1);
2583         }
2584
2585         if (i >= usec_timeout) {
2586                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2587                 return -EINVAL;
2588         }
2589
2590         return 0;
2591 }
2592
2593 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2594 {
2595         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2596         uint32_t tmp;
2597         unsigned i, pipe_id;
2598         const struct gfx_firmware_header_v2_0 *me_hdr;
2599
2600         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2601                 adev->gfx.me_fw->data;
2602
2603         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2604                 lower_32_bits(addr));
2605         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2606                 upper_32_bits(addr));
2607
2608         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2609         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2610         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2611         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2612         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2613
2614         /*
2615          * Programming any of the CP_ME_IC_BASE registers
2616          * forces invalidation of the ME L1 I$. Wait for the
2617          * invalidation complete
2618          */
2619         for (i = 0; i < usec_timeout; i++) {
2620                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2621                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2622                         INVALIDATE_CACHE_COMPLETE))
2623                         break;
2624                 udelay(1);
2625         }
2626
2627         if (i >= usec_timeout) {
2628                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2629                 return -EINVAL;
2630         }
2631
2632         /* Prime the instruction caches */
2633         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2634         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2635         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2636
2637         /* Waiting for instruction cache primed*/
2638         for (i = 0; i < usec_timeout; i++) {
2639                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2640                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2641                         ICACHE_PRIMED))
2642                         break;
2643                 udelay(1);
2644         }
2645
2646         if (i >= usec_timeout) {
2647                 dev_err(adev->dev, "failed to prime instruction cache\n");
2648                 return -EINVAL;
2649         }
2650
2651         mutex_lock(&adev->srbm_mutex);
2652         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2653                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2654                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2655                         (me_hdr->ucode_start_addr_hi << 30) |
2656                         (me_hdr->ucode_start_addr_lo >> 2) );
2657                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2658                         me_hdr->ucode_start_addr_hi>>2);
2659
2660                 /*
2661                  * Program CP_ME_CNTL to reset given PIPE to take
2662                  * effect of CP_PFP_PRGRM_CNTR_START.
2663                  */
2664                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2665                 if (pipe_id == 0)
2666                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2667                                         ME_PIPE0_RESET, 1);
2668                 else
2669                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2670                                         ME_PIPE1_RESET, 1);
2671                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2672
2673                 /* Clear pfp pipe0 reset bit. */
2674                 if (pipe_id == 0)
2675                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2676                                         ME_PIPE0_RESET, 0);
2677                 else
2678                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2679                                         ME_PIPE1_RESET, 0);
2680                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2681
2682                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2683                         lower_32_bits(addr2));
2684                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2685                         upper_32_bits(addr2));
2686         }
2687         soc21_grbm_select(adev, 0, 0, 0, 0);
2688         mutex_unlock(&adev->srbm_mutex);
2689
2690         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2691         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2692         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2693         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2694
2695         /* Invalidate the data caches */
2696         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2697         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2698         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2699
2700         for (i = 0; i < usec_timeout; i++) {
2701                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2702                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2703                         INVALIDATE_DCACHE_COMPLETE))
2704                         break;
2705                 udelay(1);
2706         }
2707
2708         if (i >= usec_timeout) {
2709                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2710                 return -EINVAL;
2711         }
2712
2713         return 0;
2714 }
2715
2716 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2717 {
2718         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2719         uint32_t tmp;
2720         unsigned i;
2721         const struct gfx_firmware_header_v2_0 *mec_hdr;
2722
2723         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2724                 adev->gfx.mec_fw->data;
2725
2726         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2727         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2728         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2729         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2730         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2731
2732         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2733         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2734         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2735         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2736
2737         mutex_lock(&adev->srbm_mutex);
2738         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2739                 soc21_grbm_select(adev, 1, i, 0, 0);
2740
2741                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2);
2742                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2743                      upper_32_bits(addr2));
2744
2745                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2746                                         mec_hdr->ucode_start_addr_lo >> 2 |
2747                                         mec_hdr->ucode_start_addr_hi << 30);
2748                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2749                                         mec_hdr->ucode_start_addr_hi >> 2);
2750
2751                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr);
2752                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2753                      upper_32_bits(addr));
2754         }
2755         mutex_unlock(&adev->srbm_mutex);
2756         soc21_grbm_select(adev, 0, 0, 0, 0);
2757
2758         /* Trigger an invalidation of the L1 instruction caches */
2759         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2760         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2761         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2762
2763         /* Wait for invalidation complete */
2764         for (i = 0; i < usec_timeout; i++) {
2765                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2766                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2767                                        INVALIDATE_DCACHE_COMPLETE))
2768                         break;
2769                 udelay(1);
2770         }
2771
2772         if (i >= usec_timeout) {
2773                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2774                 return -EINVAL;
2775         }
2776
2777         /* Trigger an invalidation of the L1 instruction caches */
2778         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2779         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2780         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2781
2782         /* Wait for invalidation complete */
2783         for (i = 0; i < usec_timeout; i++) {
2784                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2785                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2786                                        INVALIDATE_CACHE_COMPLETE))
2787                         break;
2788                 udelay(1);
2789         }
2790
2791         if (i >= usec_timeout) {
2792                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2793                 return -EINVAL;
2794         }
2795
2796         return 0;
2797 }
2798
2799 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)
2800 {
2801         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2802         const struct gfx_firmware_header_v2_0 *me_hdr;
2803         const struct gfx_firmware_header_v2_0 *mec_hdr;
2804         uint32_t pipe_id, tmp;
2805
2806         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2807                 adev->gfx.mec_fw->data;
2808         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2809                 adev->gfx.me_fw->data;
2810         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2811                 adev->gfx.pfp_fw->data;
2812
2813         /* config pfp program start addr */
2814         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2815                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2816                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2817                         (pfp_hdr->ucode_start_addr_hi << 30) |
2818                         (pfp_hdr->ucode_start_addr_lo >> 2));
2819                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2820                         pfp_hdr->ucode_start_addr_hi >> 2);
2821         }
2822         soc21_grbm_select(adev, 0, 0, 0, 0);
2823
2824         /* reset pfp pipe */
2825         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2826         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
2827         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
2828         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2829
2830         /* clear pfp pipe reset */
2831         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
2832         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
2833         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2834
2835         /* config me program start addr */
2836         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2837                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2838                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2839                         (me_hdr->ucode_start_addr_hi << 30) |
2840                         (me_hdr->ucode_start_addr_lo >> 2) );
2841                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2842                         me_hdr->ucode_start_addr_hi>>2);
2843         }
2844         soc21_grbm_select(adev, 0, 0, 0, 0);
2845
2846         /* reset me pipe */
2847         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2848         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
2849         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
2850         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2851
2852         /* clear me pipe reset */
2853         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
2854         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
2855         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2856
2857         /* config mec program start addr */
2858         for (pipe_id = 0; pipe_id < 4; pipe_id++) {
2859                 soc21_grbm_select(adev, 1, pipe_id, 0, 0);
2860                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2861                                         mec_hdr->ucode_start_addr_lo >> 2 |
2862                                         mec_hdr->ucode_start_addr_hi << 30);
2863                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2864                                         mec_hdr->ucode_start_addr_hi >> 2);
2865         }
2866         soc21_grbm_select(adev, 0, 0, 0, 0);
2867
2868         /* reset mec pipe */
2869         tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
2870         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
2871         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
2872         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
2873         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
2874         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2875
2876         /* clear mec pipe reset */
2877         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
2878         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
2879         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
2880         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
2881         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2882 }
2883
2884 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2885 {
2886         uint32_t cp_status;
2887         uint32_t bootload_status;
2888         int i, r;
2889         uint64_t addr, addr2;
2890
2891         for (i = 0; i < adev->usec_timeout; i++) {
2892                 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
2893
2894                 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
2895                             IP_VERSION(11, 0, 1) ||
2896                     amdgpu_ip_version(adev, GC_HWIP, 0) ==
2897                             IP_VERSION(11, 0, 4) ||
2898                     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) ||
2899                     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) ||
2900                     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2))
2901                         bootload_status = RREG32_SOC15(GC, 0,
2902                                         regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1);
2903                 else
2904                         bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
2905
2906                 if ((cp_status == 0) &&
2907                     (REG_GET_FIELD(bootload_status,
2908                         RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
2909                         break;
2910                 }
2911                 udelay(1);
2912         }
2913
2914         if (i >= adev->usec_timeout) {
2915                 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
2916                 return -ETIMEDOUT;
2917         }
2918
2919         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2920                 if (adev->gfx.rs64_enable) {
2921                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2922                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset;
2923                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2924                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset;
2925                         r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2);
2926                         if (r)
2927                                 return r;
2928                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2929                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset;
2930                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2931                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset;
2932                         r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2);
2933                         if (r)
2934                                 return r;
2935                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2936                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset;
2937                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2938                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset;
2939                         r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2);
2940                         if (r)
2941                                 return r;
2942                 } else {
2943                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2944                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset;
2945                         r = gfx_v11_0_config_me_cache(adev, addr);
2946                         if (r)
2947                                 return r;
2948                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2949                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset;
2950                         r = gfx_v11_0_config_pfp_cache(adev, addr);
2951                         if (r)
2952                                 return r;
2953                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2954                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset;
2955                         r = gfx_v11_0_config_mec_cache(adev, addr);
2956                         if (r)
2957                                 return r;
2958                 }
2959         }
2960
2961         return 0;
2962 }
2963
2964 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2965 {
2966         int i;
2967         u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2968
2969         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2970         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2971         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2972
2973         for (i = 0; i < adev->usec_timeout; i++) {
2974                 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
2975                         break;
2976                 udelay(1);
2977         }
2978
2979         if (i >= adev->usec_timeout)
2980                 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
2981
2982         return 0;
2983 }
2984
2985 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
2986 {
2987         int r;
2988         const struct gfx_firmware_header_v1_0 *pfp_hdr;
2989         const __le32 *fw_data;
2990         unsigned i, fw_size;
2991
2992         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2993                 adev->gfx.pfp_fw->data;
2994
2995         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2996
2997         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2998                 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2999         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
3000
3001         r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
3002                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3003                                       &adev->gfx.pfp.pfp_fw_obj,
3004                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
3005                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
3006         if (r) {
3007                 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
3008                 gfx_v11_0_pfp_fini(adev);
3009                 return r;
3010         }
3011
3012         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
3013
3014         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
3015         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
3016
3017         gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr);
3018
3019         WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0);
3020
3021         for (i = 0; i < pfp_hdr->jt_size; i++)
3022                 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA,
3023                              le32_to_cpup(fw_data + pfp_hdr->jt_offset + i));
3024
3025         WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3026
3027         return 0;
3028 }
3029
3030 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
3031 {
3032         int r;
3033         const struct gfx_firmware_header_v2_0 *pfp_hdr;
3034         const __le32 *fw_ucode, *fw_data;
3035         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3036         uint32_t tmp;
3037         uint32_t usec_timeout = 50000;  /* wait for 50ms */
3038
3039         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
3040                 adev->gfx.pfp_fw->data;
3041
3042         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3043
3044         /* instruction */
3045         fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
3046                 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
3047         fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
3048         /* data */
3049         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
3050                 le32_to_cpu(pfp_hdr->data_offset_bytes));
3051         fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
3052
3053         /* 64kb align */
3054         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3055                                       64 * 1024,
3056                                       AMDGPU_GEM_DOMAIN_VRAM |
3057                                       AMDGPU_GEM_DOMAIN_GTT,
3058                                       &adev->gfx.pfp.pfp_fw_obj,
3059                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
3060                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
3061         if (r) {
3062                 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
3063                 gfx_v11_0_pfp_fini(adev);
3064                 return r;
3065         }
3066
3067         r = amdgpu_bo_create_reserved(adev, fw_data_size,
3068                                       64 * 1024,
3069                                       AMDGPU_GEM_DOMAIN_VRAM |
3070                                       AMDGPU_GEM_DOMAIN_GTT,
3071                                       &adev->gfx.pfp.pfp_fw_data_obj,
3072                                       &adev->gfx.pfp.pfp_fw_data_gpu_addr,
3073                                       (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
3074         if (r) {
3075                 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
3076                 gfx_v11_0_pfp_fini(adev);
3077                 return r;
3078         }
3079
3080         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
3081         memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
3082
3083         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
3084         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
3085         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
3086         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
3087
3088         if (amdgpu_emu_mode == 1)
3089                 adev->hdp.funcs->flush_hdp(adev, NULL);
3090
3091         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
3092                 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
3093         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
3094                 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
3095
3096         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
3097         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
3098         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
3099         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
3100         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
3101
3102         /*
3103          * Programming any of the CP_PFP_IC_BASE registers
3104          * forces invalidation of the ME L1 I$. Wait for the
3105          * invalidation complete
3106          */
3107         for (i = 0; i < usec_timeout; i++) {
3108                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3109                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3110                         INVALIDATE_CACHE_COMPLETE))
3111                         break;
3112                 udelay(1);
3113         }
3114
3115         if (i >= usec_timeout) {
3116                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3117                 return -EINVAL;
3118         }
3119
3120         /* Prime the L1 instruction caches */
3121         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3122         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
3123         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
3124         /* Waiting for cache primed*/
3125         for (i = 0; i < usec_timeout; i++) {
3126                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3127                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3128                         ICACHE_PRIMED))
3129                         break;
3130                 udelay(1);
3131         }
3132
3133         if (i >= usec_timeout) {
3134                 dev_err(adev->dev, "failed to prime instruction cache\n");
3135                 return -EINVAL;
3136         }
3137
3138         mutex_lock(&adev->srbm_mutex);
3139         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3140                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3141                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
3142                         (pfp_hdr->ucode_start_addr_hi << 30) |
3143                         (pfp_hdr->ucode_start_addr_lo >> 2) );
3144                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
3145                         pfp_hdr->ucode_start_addr_hi>>2);
3146
3147                 /*
3148                  * Program CP_ME_CNTL to reset given PIPE to take
3149                  * effect of CP_PFP_PRGRM_CNTR_START.
3150                  */
3151                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3152                 if (pipe_id == 0)
3153                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3154                                         PFP_PIPE0_RESET, 1);
3155                 else
3156                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3157                                         PFP_PIPE1_RESET, 1);
3158                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3159
3160                 /* Clear pfp pipe0 reset bit. */
3161                 if (pipe_id == 0)
3162                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3163                                         PFP_PIPE0_RESET, 0);
3164                 else
3165                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3166                                         PFP_PIPE1_RESET, 0);
3167                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3168
3169                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
3170                         lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3171                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
3172                         upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3173         }
3174         soc21_grbm_select(adev, 0, 0, 0, 0);
3175         mutex_unlock(&adev->srbm_mutex);
3176
3177         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3178         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3179         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3180         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3181
3182         /* Invalidate the data caches */
3183         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3184         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3185         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3186
3187         for (i = 0; i < usec_timeout; i++) {
3188                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3189                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3190                         INVALIDATE_DCACHE_COMPLETE))
3191                         break;
3192                 udelay(1);
3193         }
3194
3195         if (i >= usec_timeout) {
3196                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3197                 return -EINVAL;
3198         }
3199
3200         return 0;
3201 }
3202
3203 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
3204 {
3205         int r;
3206         const struct gfx_firmware_header_v1_0 *me_hdr;
3207         const __le32 *fw_data;
3208         unsigned i, fw_size;
3209
3210         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3211                 adev->gfx.me_fw->data;
3212
3213         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3214
3215         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3216                 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3217         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
3218
3219         r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
3220                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3221                                       &adev->gfx.me.me_fw_obj,
3222                                       &adev->gfx.me.me_fw_gpu_addr,
3223                                       (void **)&adev->gfx.me.me_fw_ptr);
3224         if (r) {
3225                 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
3226                 gfx_v11_0_me_fini(adev);
3227                 return r;
3228         }
3229
3230         memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
3231
3232         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3233         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3234
3235         gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr);
3236
3237         WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0);
3238
3239         for (i = 0; i < me_hdr->jt_size; i++)
3240                 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA,
3241                              le32_to_cpup(fw_data + me_hdr->jt_offset + i));
3242
3243         WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version);
3244
3245         return 0;
3246 }
3247
3248 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
3249 {
3250         int r;
3251         const struct gfx_firmware_header_v2_0 *me_hdr;
3252         const __le32 *fw_ucode, *fw_data;
3253         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3254         uint32_t tmp;
3255         uint32_t usec_timeout = 50000;  /* wait for 50ms */
3256
3257         me_hdr = (const struct gfx_firmware_header_v2_0 *)
3258                 adev->gfx.me_fw->data;
3259
3260         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3261
3262         /* instruction */
3263         fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
3264                 le32_to_cpu(me_hdr->ucode_offset_bytes));
3265         fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
3266         /* data */
3267         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3268                 le32_to_cpu(me_hdr->data_offset_bytes));
3269         fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
3270
3271         /* 64kb align*/
3272         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3273                                       64 * 1024,
3274                                       AMDGPU_GEM_DOMAIN_VRAM |
3275                                       AMDGPU_GEM_DOMAIN_GTT,
3276                                       &adev->gfx.me.me_fw_obj,
3277                                       &adev->gfx.me.me_fw_gpu_addr,
3278                                       (void **)&adev->gfx.me.me_fw_ptr);
3279         if (r) {
3280                 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
3281                 gfx_v11_0_me_fini(adev);
3282                 return r;
3283         }
3284
3285         r = amdgpu_bo_create_reserved(adev, fw_data_size,
3286                                       64 * 1024,
3287                                       AMDGPU_GEM_DOMAIN_VRAM |
3288                                       AMDGPU_GEM_DOMAIN_GTT,
3289                                       &adev->gfx.me.me_fw_data_obj,
3290                                       &adev->gfx.me.me_fw_data_gpu_addr,
3291                                       (void **)&adev->gfx.me.me_fw_data_ptr);
3292         if (r) {
3293                 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
3294                 gfx_v11_0_pfp_fini(adev);
3295                 return r;
3296         }
3297
3298         memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
3299         memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
3300
3301         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3302         amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
3303         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3304         amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
3305
3306         if (amdgpu_emu_mode == 1)
3307                 adev->hdp.funcs->flush_hdp(adev, NULL);
3308
3309         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
3310                 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
3311         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
3312                 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
3313
3314         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
3315         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
3316         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
3317         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
3318         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
3319
3320         /*
3321          * Programming any of the CP_ME_IC_BASE registers
3322          * forces invalidation of the ME L1 I$. Wait for the
3323          * invalidation complete
3324          */
3325         for (i = 0; i < usec_timeout; i++) {
3326                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3327                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3328                         INVALIDATE_CACHE_COMPLETE))
3329                         break;
3330                 udelay(1);
3331         }
3332
3333         if (i >= usec_timeout) {
3334                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3335                 return -EINVAL;
3336         }
3337
3338         /* Prime the instruction caches */
3339         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3340         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
3341         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
3342
3343         /* Waiting for instruction cache primed*/
3344         for (i = 0; i < usec_timeout; i++) {
3345                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3346                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3347                         ICACHE_PRIMED))
3348                         break;
3349                 udelay(1);
3350         }
3351
3352         if (i >= usec_timeout) {
3353                 dev_err(adev->dev, "failed to prime instruction cache\n");
3354                 return -EINVAL;
3355         }
3356
3357         mutex_lock(&adev->srbm_mutex);
3358         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3359                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3360                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
3361                         (me_hdr->ucode_start_addr_hi << 30) |
3362                         (me_hdr->ucode_start_addr_lo >> 2) );
3363                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
3364                         me_hdr->ucode_start_addr_hi>>2);
3365
3366                 /*
3367                  * Program CP_ME_CNTL to reset given PIPE to take
3368                  * effect of CP_PFP_PRGRM_CNTR_START.
3369                  */
3370                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3371                 if (pipe_id == 0)
3372                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3373                                         ME_PIPE0_RESET, 1);
3374                 else
3375                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3376                                         ME_PIPE1_RESET, 1);
3377                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3378
3379                 /* Clear pfp pipe0 reset bit. */
3380                 if (pipe_id == 0)
3381                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3382                                         ME_PIPE0_RESET, 0);
3383                 else
3384                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3385                                         ME_PIPE1_RESET, 0);
3386                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3387
3388                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
3389                         lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3390                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
3391                         upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3392         }
3393         soc21_grbm_select(adev, 0, 0, 0, 0);
3394         mutex_unlock(&adev->srbm_mutex);
3395
3396         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3397         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3398         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3399         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3400
3401         /* Invalidate the data caches */
3402         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3403         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3404         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3405
3406         for (i = 0; i < usec_timeout; i++) {
3407                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3408                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3409                         INVALIDATE_DCACHE_COMPLETE))
3410                         break;
3411                 udelay(1);
3412         }
3413
3414         if (i >= usec_timeout) {
3415                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3416                 return -EINVAL;
3417         }
3418
3419         return 0;
3420 }
3421
3422 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3423 {
3424         int r;
3425
3426         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
3427                 return -EINVAL;
3428
3429         gfx_v11_0_cp_gfx_enable(adev, false);
3430
3431         if (adev->gfx.rs64_enable)
3432                 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev);
3433         else
3434                 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev);
3435         if (r) {
3436                 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
3437                 return r;
3438         }
3439
3440         if (adev->gfx.rs64_enable)
3441                 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev);
3442         else
3443                 r = gfx_v11_0_cp_gfx_load_me_microcode(adev);
3444         if (r) {
3445                 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
3446                 return r;
3447         }
3448
3449         return 0;
3450 }
3451
3452 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev)
3453 {
3454         struct amdgpu_ring *ring;
3455         const struct cs_section_def *sect = NULL;
3456         const struct cs_extent_def *ext = NULL;
3457         int r, i;
3458         int ctx_reg_offset;
3459
3460         /* init the CP */
3461         WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
3462                      adev->gfx.config.max_hw_contexts - 1);
3463         WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
3464
3465         if (!amdgpu_async_gfx_ring)
3466                 gfx_v11_0_cp_gfx_enable(adev, true);
3467
3468         ring = &adev->gfx.gfx_ring[0];
3469         r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev));
3470         if (r) {
3471                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3472                 return r;
3473         }
3474
3475         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3476         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3477
3478         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3479         amdgpu_ring_write(ring, 0x80000000);
3480         amdgpu_ring_write(ring, 0x80000000);
3481
3482         for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
3483                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3484                         if (sect->id == SECT_CONTEXT) {
3485                                 amdgpu_ring_write(ring,
3486                                                   PACKET3(PACKET3_SET_CONTEXT_REG,
3487                                                           ext->reg_count));
3488                                 amdgpu_ring_write(ring, ext->reg_index -
3489                                                   PACKET3_SET_CONTEXT_REG_START);
3490                                 for (i = 0; i < ext->reg_count; i++)
3491                                         amdgpu_ring_write(ring, ext->extent[i]);
3492                         }
3493                 }
3494         }
3495
3496         ctx_reg_offset =
3497                 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
3498         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
3499         amdgpu_ring_write(ring, ctx_reg_offset);
3500         amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
3501
3502         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3503         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3504
3505         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3506         amdgpu_ring_write(ring, 0);
3507
3508         amdgpu_ring_commit(ring);
3509
3510         /* submit cs packet to copy state 0 to next available state */
3511         if (adev->gfx.num_gfx_rings > 1) {
3512                 /* maximum supported gfx ring is 2 */
3513                 ring = &adev->gfx.gfx_ring[1];
3514                 r = amdgpu_ring_alloc(ring, 2);
3515                 if (r) {
3516                         DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3517                         return r;
3518                 }
3519
3520                 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3521                 amdgpu_ring_write(ring, 0);
3522
3523                 amdgpu_ring_commit(ring);
3524         }
3525         return 0;
3526 }
3527
3528 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
3529                                          CP_PIPE_ID pipe)
3530 {
3531         u32 tmp;
3532
3533         tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
3534         tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
3535
3536         WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
3537 }
3538
3539 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
3540                                           struct amdgpu_ring *ring)
3541 {
3542         u32 tmp;
3543
3544         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3545         if (ring->use_doorbell) {
3546                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3547                                     DOORBELL_OFFSET, ring->doorbell_index);
3548                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3549                                     DOORBELL_EN, 1);
3550         } else {
3551                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3552                                     DOORBELL_EN, 0);
3553         }
3554         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
3555
3556         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3557                             DOORBELL_RANGE_LOWER, ring->doorbell_index);
3558         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
3559
3560         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3561                      CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3562 }
3563
3564 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev)
3565 {
3566         struct amdgpu_ring *ring;
3567         u32 tmp;
3568         u32 rb_bufsz;
3569         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3570
3571         /* Set the write pointer delay */
3572         WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
3573
3574         /* set the RB to use vmid 0 */
3575         WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
3576
3577         /* Init gfx ring 0 for pipe 0 */
3578         mutex_lock(&adev->srbm_mutex);
3579         gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3580
3581         /* Set ring buffer size */
3582         ring = &adev->gfx.gfx_ring[0];
3583         rb_bufsz = order_base_2(ring->ring_size / 8);
3584         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3585         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3586         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3587
3588         /* Initialize the ring buffer's write pointers */
3589         ring->wptr = 0;
3590         WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
3591         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3592
3593         /* set the wb address whether it's enabled or not */
3594         rptr_addr = ring->rptr_gpu_addr;
3595         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3596         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3597                      CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3598
3599         wptr_gpu_addr = ring->wptr_gpu_addr;
3600         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3601                      lower_32_bits(wptr_gpu_addr));
3602         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3603                      upper_32_bits(wptr_gpu_addr));
3604
3605         mdelay(1);
3606         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3607
3608         rb_addr = ring->gpu_addr >> 8;
3609         WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
3610         WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3611
3612         WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
3613
3614         gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3615         mutex_unlock(&adev->srbm_mutex);
3616
3617         /* Init gfx ring 1 for pipe 1 */
3618         if (adev->gfx.num_gfx_rings > 1) {
3619                 mutex_lock(&adev->srbm_mutex);
3620                 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
3621                 /* maximum supported gfx ring is 2 */
3622                 ring = &adev->gfx.gfx_ring[1];
3623                 rb_bufsz = order_base_2(ring->ring_size / 8);
3624                 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
3625                 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
3626                 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3627                 /* Initialize the ring buffer's write pointers */
3628                 ring->wptr = 0;
3629                 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr));
3630                 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
3631                 /* Set the wb address whether it's enabled or not */
3632                 rptr_addr = ring->rptr_gpu_addr;
3633                 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
3634                 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3635                              CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3636                 wptr_gpu_addr = ring->wptr_gpu_addr;
3637                 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3638                              lower_32_bits(wptr_gpu_addr));
3639                 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3640                              upper_32_bits(wptr_gpu_addr));
3641
3642                 mdelay(1);
3643                 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3644
3645                 rb_addr = ring->gpu_addr >> 8;
3646                 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr);
3647                 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr));
3648                 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1);
3649
3650                 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3651                 mutex_unlock(&adev->srbm_mutex);
3652         }
3653         /* Switch to pipe 0 */
3654         mutex_lock(&adev->srbm_mutex);
3655         gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3656         mutex_unlock(&adev->srbm_mutex);
3657
3658         /* start the ring */
3659         gfx_v11_0_cp_gfx_start(adev);
3660
3661         return 0;
3662 }
3663
3664 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3665 {
3666         u32 data;
3667
3668         if (adev->gfx.rs64_enable) {
3669                 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3670                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
3671                                                          enable ? 0 : 1);
3672                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
3673                                                          enable ? 0 : 1);
3674                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
3675                                                          enable ? 0 : 1);
3676                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
3677                                                          enable ? 0 : 1);
3678                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
3679                                                          enable ? 0 : 1);
3680                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
3681                                                          enable ? 1 : 0);
3682                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
3683                                                          enable ? 1 : 0);
3684                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
3685                                                          enable ? 1 : 0);
3686                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
3687                                                          enable ? 1 : 0);
3688                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
3689                                                          enable ? 0 : 1);
3690                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
3691         } else {
3692                 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
3693
3694                 if (enable) {
3695                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0);
3696                         if (!adev->enable_mes_kiq)
3697                                 data = REG_SET_FIELD(data, CP_MEC_CNTL,
3698                                                      MEC_ME2_HALT, 0);
3699                 } else {
3700                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1);
3701                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1);
3702                 }
3703                 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data);
3704         }
3705
3706         udelay(50);
3707 }
3708
3709 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3710 {
3711         const struct gfx_firmware_header_v1_0 *mec_hdr;
3712         const __le32 *fw_data;
3713         unsigned i, fw_size;
3714         u32 *fw = NULL;
3715         int r;
3716
3717         if (!adev->gfx.mec_fw)
3718                 return -EINVAL;
3719
3720         gfx_v11_0_cp_compute_enable(adev, false);
3721
3722         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3723         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3724
3725         fw_data = (const __le32 *)
3726                 (adev->gfx.mec_fw->data +
3727                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3728         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
3729
3730         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
3731                                           PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3732                                           &adev->gfx.mec.mec_fw_obj,
3733                                           &adev->gfx.mec.mec_fw_gpu_addr,
3734                                           (void **)&fw);
3735         if (r) {
3736                 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
3737                 gfx_v11_0_mec_fini(adev);
3738                 return r;
3739         }
3740
3741         memcpy(fw, fw_data, fw_size);
3742
3743         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3744         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3745
3746         gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr);
3747
3748         /* MEC1 */
3749         WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0);
3750
3751         for (i = 0; i < mec_hdr->jt_size; i++)
3752                 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA,
3753                              le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3754
3755         WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
3756
3757         return 0;
3758 }
3759
3760 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
3761 {
3762         const struct gfx_firmware_header_v2_0 *mec_hdr;
3763         const __le32 *fw_ucode, *fw_data;
3764         u32 tmp, fw_ucode_size, fw_data_size;
3765         u32 i, usec_timeout = 50000; /* Wait for 50 ms */
3766         u32 *fw_ucode_ptr, *fw_data_ptr;
3767         int r;
3768
3769         if (!adev->gfx.mec_fw)
3770                 return -EINVAL;
3771
3772         gfx_v11_0_cp_compute_enable(adev, false);
3773
3774         mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
3775         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3776
3777         fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
3778                                 le32_to_cpu(mec_hdr->ucode_offset_bytes));
3779         fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
3780
3781         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
3782                                 le32_to_cpu(mec_hdr->data_offset_bytes));
3783         fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
3784
3785         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3786                                       64 * 1024,
3787                                       AMDGPU_GEM_DOMAIN_VRAM |
3788                                       AMDGPU_GEM_DOMAIN_GTT,
3789                                       &adev->gfx.mec.mec_fw_obj,
3790                                       &adev->gfx.mec.mec_fw_gpu_addr,
3791                                       (void **)&fw_ucode_ptr);
3792         if (r) {
3793                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3794                 gfx_v11_0_mec_fini(adev);
3795                 return r;
3796         }
3797
3798         r = amdgpu_bo_create_reserved(adev, fw_data_size,
3799                                       64 * 1024,
3800                                       AMDGPU_GEM_DOMAIN_VRAM |
3801                                       AMDGPU_GEM_DOMAIN_GTT,
3802                                       &adev->gfx.mec.mec_fw_data_obj,
3803                                       &adev->gfx.mec.mec_fw_data_gpu_addr,
3804                                       (void **)&fw_data_ptr);
3805         if (r) {
3806                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3807                 gfx_v11_0_mec_fini(adev);
3808                 return r;
3809         }
3810
3811         memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
3812         memcpy(fw_data_ptr, fw_data, fw_data_size);
3813
3814         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3815         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
3816         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3817         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
3818
3819         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
3820         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3821         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
3822         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3823         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
3824
3825         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
3826         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
3827         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
3828         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
3829
3830         mutex_lock(&adev->srbm_mutex);
3831         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
3832                 soc21_grbm_select(adev, 1, i, 0, 0);
3833
3834                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr);
3835                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
3836                      upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr));
3837
3838                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
3839                                         mec_hdr->ucode_start_addr_lo >> 2 |
3840                                         mec_hdr->ucode_start_addr_hi << 30);
3841                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
3842                                         mec_hdr->ucode_start_addr_hi >> 2);
3843
3844                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr);
3845                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
3846                      upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3847         }
3848         mutex_unlock(&adev->srbm_mutex);
3849         soc21_grbm_select(adev, 0, 0, 0, 0);
3850
3851         /* Trigger an invalidation of the L1 instruction caches */
3852         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3853         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3854         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
3855
3856         /* Wait for invalidation complete */
3857         for (i = 0; i < usec_timeout; i++) {
3858                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3859                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
3860                                        INVALIDATE_DCACHE_COMPLETE))
3861                         break;
3862                 udelay(1);
3863         }
3864
3865         if (i >= usec_timeout) {
3866                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3867                 return -EINVAL;
3868         }
3869
3870         /* Trigger an invalidation of the L1 instruction caches */
3871         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3872         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
3873         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
3874
3875         /* Wait for invalidation complete */
3876         for (i = 0; i < usec_timeout; i++) {
3877                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3878                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
3879                                        INVALIDATE_CACHE_COMPLETE))
3880                         break;
3881                 udelay(1);
3882         }
3883
3884         if (i >= usec_timeout) {
3885                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3886                 return -EINVAL;
3887         }
3888
3889         return 0;
3890 }
3891
3892 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring)
3893 {
3894         uint32_t tmp;
3895         struct amdgpu_device *adev = ring->adev;
3896
3897         /* tell RLC which is KIQ queue */
3898         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
3899         tmp &= 0xffffff00;
3900         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3901         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3902         tmp |= 0x80;
3903         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3904 }
3905
3906 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev)
3907 {
3908         /* set graphics engine doorbell range */
3909         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
3910                      (adev->doorbell_index.gfx_ring0 * 2) << 2);
3911         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3912                      (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
3913
3914         /* set compute engine doorbell range */
3915         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
3916                      (adev->doorbell_index.kiq * 2) << 2);
3917         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
3918                      (adev->doorbell_index.userqueue_end * 2) << 2);
3919 }
3920
3921 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev,
3922                                            struct v11_gfx_mqd *mqd,
3923                                            struct amdgpu_mqd_prop *prop)
3924 {
3925         bool priority = 0;
3926         u32 tmp;
3927
3928         /* set up default queue priority level
3929          * 0x0 = low priority, 0x1 = high priority
3930          */
3931         if (prop->hqd_pipe_priority == AMDGPU_GFX_PIPE_PRIO_HIGH)
3932                 priority = 1;
3933
3934         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY);
3935         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority);
3936         mqd->cp_gfx_hqd_queue_priority = tmp;
3937 }
3938
3939 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
3940                                   struct amdgpu_mqd_prop *prop)
3941 {
3942         struct v11_gfx_mqd *mqd = m;
3943         uint64_t hqd_gpu_addr, wb_gpu_addr;
3944         uint32_t tmp;
3945         uint32_t rb_bufsz;
3946
3947         /* set up gfx hqd wptr */
3948         mqd->cp_gfx_hqd_wptr = 0;
3949         mqd->cp_gfx_hqd_wptr_hi = 0;
3950
3951         /* set the pointer to the MQD */
3952         mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
3953         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
3954
3955         /* set up mqd control */
3956         tmp = RREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL);
3957         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
3958         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
3959         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
3960         mqd->cp_gfx_mqd_control = tmp;
3961
3962         /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
3963         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID);
3964         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
3965         mqd->cp_gfx_hqd_vmid = 0;
3966
3967         /* set up gfx queue priority */
3968         gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop);
3969
3970         /* set up time quantum */
3971         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM);
3972         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
3973         mqd->cp_gfx_hqd_quantum = tmp;
3974
3975         /* set up gfx hqd base. this is similar as CP_RB_BASE */
3976         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
3977         mqd->cp_gfx_hqd_base = hqd_gpu_addr;
3978         mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
3979
3980         /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
3981         wb_gpu_addr = prop->rptr_gpu_addr;
3982         mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
3983         mqd->cp_gfx_hqd_rptr_addr_hi =
3984                 upper_32_bits(wb_gpu_addr) & 0xffff;
3985
3986         /* set up rb_wptr_poll addr */
3987         wb_gpu_addr = prop->wptr_gpu_addr;
3988         mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3989         mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3990
3991         /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
3992         rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
3993         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL);
3994         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
3995         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
3996 #ifdef __BIG_ENDIAN
3997         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
3998 #endif
3999         mqd->cp_gfx_hqd_cntl = tmp;
4000
4001         /* set up cp_doorbell_control */
4002         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
4003         if (prop->use_doorbell) {
4004                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4005                                     DOORBELL_OFFSET, prop->doorbell_index);
4006                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4007                                     DOORBELL_EN, 1);
4008         } else
4009                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4010                                     DOORBELL_EN, 0);
4011         mqd->cp_rb_doorbell_control = tmp;
4012
4013         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4014         mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR);
4015
4016         /* active the queue */
4017         mqd->cp_gfx_hqd_active = 1;
4018
4019         return 0;
4020 }
4021
4022 static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset)
4023 {
4024         struct amdgpu_device *adev = ring->adev;
4025         struct v11_gfx_mqd *mqd = ring->mqd_ptr;
4026         int mqd_idx = ring - &adev->gfx.gfx_ring[0];
4027
4028         if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
4029                 memset((void *)mqd, 0, sizeof(*mqd));
4030                 mutex_lock(&adev->srbm_mutex);
4031                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4032                 amdgpu_ring_init_mqd(ring);
4033                 soc21_grbm_select(adev, 0, 0, 0, 0);
4034                 mutex_unlock(&adev->srbm_mutex);
4035                 if (adev->gfx.me.mqd_backup[mqd_idx])
4036                         memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4037         } else {
4038                 /* restore mqd with the backup copy */
4039                 if (adev->gfx.me.mqd_backup[mqd_idx])
4040                         memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
4041                 /* reset the ring */
4042                 ring->wptr = 0;
4043                 *ring->wptr_cpu_addr = 0;
4044                 amdgpu_ring_clear_ring(ring);
4045         }
4046
4047         return 0;
4048 }
4049
4050 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
4051 {
4052         int r, i;
4053         struct amdgpu_ring *ring;
4054
4055         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4056                 ring = &adev->gfx.gfx_ring[i];
4057
4058                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4059                 if (unlikely(r != 0))
4060                         return r;
4061
4062                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4063                 if (!r) {
4064                         r = gfx_v11_0_kgq_init_queue(ring, false);
4065                         amdgpu_bo_kunmap(ring->mqd_obj);
4066                         ring->mqd_ptr = NULL;
4067                 }
4068                 amdgpu_bo_unreserve(ring->mqd_obj);
4069                 if (r)
4070                         return r;
4071         }
4072
4073         r = amdgpu_gfx_enable_kgq(adev, 0);
4074         if (r)
4075                 return r;
4076
4077         return gfx_v11_0_cp_gfx_start(adev);
4078 }
4079
4080 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
4081                                       struct amdgpu_mqd_prop *prop)
4082 {
4083         struct v11_compute_mqd *mqd = m;
4084         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4085         uint32_t tmp;
4086
4087         mqd->header = 0xC0310800;
4088         mqd->compute_pipelinestat_enable = 0x00000001;
4089         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4090         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4091         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4092         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4093         mqd->compute_misc_reserved = 0x00000007;
4094
4095         eop_base_addr = prop->eop_gpu_addr >> 8;
4096         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4097         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4098
4099         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4100         tmp = RREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL);
4101         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4102                         (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1));
4103
4104         mqd->cp_hqd_eop_control = tmp;
4105
4106         /* enable doorbell? */
4107         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
4108
4109         if (prop->use_doorbell) {
4110                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4111                                     DOORBELL_OFFSET, prop->doorbell_index);
4112                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4113                                     DOORBELL_EN, 1);
4114                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4115                                     DOORBELL_SOURCE, 0);
4116                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4117                                     DOORBELL_HIT, 0);
4118         } else {
4119                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4120                                     DOORBELL_EN, 0);
4121         }
4122
4123         mqd->cp_hqd_pq_doorbell_control = tmp;
4124
4125         /* disable the queue if it's active */
4126         mqd->cp_hqd_dequeue_request = 0;
4127         mqd->cp_hqd_pq_rptr = 0;
4128         mqd->cp_hqd_pq_wptr_lo = 0;
4129         mqd->cp_hqd_pq_wptr_hi = 0;
4130
4131         /* set the pointer to the MQD */
4132         mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
4133         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
4134
4135         /* set MQD vmid to 0 */
4136         tmp = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
4137         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4138         mqd->cp_mqd_control = tmp;
4139
4140         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4141         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
4142         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4143         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4144
4145         /* set up the HQD, this is similar to CP_RB0_CNTL */
4146         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL);
4147         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4148                             (order_base_2(prop->queue_size / 4) - 1));
4149         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4150                             (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
4151         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
4152         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH,
4153                             prop->allow_tunneling);
4154         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4155         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4156         mqd->cp_hqd_pq_control = tmp;
4157
4158         /* set the wb address whether it's enabled or not */
4159         wb_gpu_addr = prop->rptr_gpu_addr;
4160         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4161         mqd->cp_hqd_pq_rptr_report_addr_hi =
4162                 upper_32_bits(wb_gpu_addr) & 0xffff;
4163
4164         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4165         wb_gpu_addr = prop->wptr_gpu_addr;
4166         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4167         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4168
4169         tmp = 0;
4170         /* enable the doorbell if requested */
4171         if (prop->use_doorbell) {
4172                 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
4173                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4174                                 DOORBELL_OFFSET, prop->doorbell_index);
4175
4176                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4177                                     DOORBELL_EN, 1);
4178                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4179                                     DOORBELL_SOURCE, 0);
4180                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4181                                     DOORBELL_HIT, 0);
4182         }
4183
4184         mqd->cp_hqd_pq_doorbell_control = tmp;
4185
4186         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4187         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR);
4188
4189         /* set the vmid for the queue */
4190         mqd->cp_hqd_vmid = 0;
4191
4192         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE);
4193         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
4194         mqd->cp_hqd_persistent_state = tmp;
4195
4196         /* set MIN_IB_AVAIL_SIZE */
4197         tmp = RREG32_SOC15(GC, 0, regCP_HQD_IB_CONTROL);
4198         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4199         mqd->cp_hqd_ib_control = tmp;
4200
4201         /* set static priority for a compute queue/ring */
4202         mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
4203         mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
4204
4205         mqd->cp_hqd_active = prop->hqd_active;
4206
4207         return 0;
4208 }
4209
4210 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring)
4211 {
4212         struct amdgpu_device *adev = ring->adev;
4213         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4214         int j;
4215
4216         /* inactivate the queue */
4217         if (amdgpu_sriov_vf(adev))
4218                 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
4219
4220         /* disable wptr polling */
4221         WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4222
4223         /* write the EOP addr */
4224         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
4225                mqd->cp_hqd_eop_base_addr_lo);
4226         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
4227                mqd->cp_hqd_eop_base_addr_hi);
4228
4229         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4230         WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
4231                mqd->cp_hqd_eop_control);
4232
4233         /* enable doorbell? */
4234         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4235                mqd->cp_hqd_pq_doorbell_control);
4236
4237         /* disable the queue if it's active */
4238         if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
4239                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
4240                 for (j = 0; j < adev->usec_timeout; j++) {
4241                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
4242                                 break;
4243                         udelay(1);
4244                 }
4245                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
4246                        mqd->cp_hqd_dequeue_request);
4247                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
4248                        mqd->cp_hqd_pq_rptr);
4249                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4250                        mqd->cp_hqd_pq_wptr_lo);
4251                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4252                        mqd->cp_hqd_pq_wptr_hi);
4253         }
4254
4255         /* set the pointer to the MQD */
4256         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
4257                mqd->cp_mqd_base_addr_lo);
4258         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
4259                mqd->cp_mqd_base_addr_hi);
4260
4261         /* set MQD vmid to 0 */
4262         WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
4263                mqd->cp_mqd_control);
4264
4265         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4266         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
4267                mqd->cp_hqd_pq_base_lo);
4268         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
4269                mqd->cp_hqd_pq_base_hi);
4270
4271         /* set up the HQD, this is similar to CP_RB0_CNTL */
4272         WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
4273                mqd->cp_hqd_pq_control);
4274
4275         /* set the wb address whether it's enabled or not */
4276         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
4277                 mqd->cp_hqd_pq_rptr_report_addr_lo);
4278         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4279                 mqd->cp_hqd_pq_rptr_report_addr_hi);
4280
4281         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4282         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
4283                mqd->cp_hqd_pq_wptr_poll_addr_lo);
4284         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
4285                mqd->cp_hqd_pq_wptr_poll_addr_hi);
4286
4287         /* enable the doorbell if requested */
4288         if (ring->use_doorbell) {
4289                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
4290                         (adev->doorbell_index.kiq * 2) << 2);
4291                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
4292                         (adev->doorbell_index.userqueue_end * 2) << 2);
4293         }
4294
4295         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4296                mqd->cp_hqd_pq_doorbell_control);
4297
4298         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4299         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4300                mqd->cp_hqd_pq_wptr_lo);
4301         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4302                mqd->cp_hqd_pq_wptr_hi);
4303
4304         /* set the vmid for the queue */
4305         WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
4306
4307         WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
4308                mqd->cp_hqd_persistent_state);
4309
4310         /* activate the queue */
4311         WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
4312                mqd->cp_hqd_active);
4313
4314         if (ring->use_doorbell)
4315                 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4316
4317         return 0;
4318 }
4319
4320 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)
4321 {
4322         struct amdgpu_device *adev = ring->adev;
4323         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4324
4325         gfx_v11_0_kiq_setting(ring);
4326
4327         if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4328                 /* reset MQD to a clean status */
4329                 if (adev->gfx.kiq[0].mqd_backup)
4330                         memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd));
4331
4332                 /* reset ring buffer */
4333                 ring->wptr = 0;
4334                 amdgpu_ring_clear_ring(ring);
4335
4336                 mutex_lock(&adev->srbm_mutex);
4337                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4338                 gfx_v11_0_kiq_init_register(ring);
4339                 soc21_grbm_select(adev, 0, 0, 0, 0);
4340                 mutex_unlock(&adev->srbm_mutex);
4341         } else {
4342                 memset((void *)mqd, 0, sizeof(*mqd));
4343                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4344                         amdgpu_ring_clear_ring(ring);
4345                 mutex_lock(&adev->srbm_mutex);
4346                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4347                 amdgpu_ring_init_mqd(ring);
4348                 gfx_v11_0_kiq_init_register(ring);
4349                 soc21_grbm_select(adev, 0, 0, 0, 0);
4350                 mutex_unlock(&adev->srbm_mutex);
4351
4352                 if (adev->gfx.kiq[0].mqd_backup)
4353                         memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd));
4354         }
4355
4356         return 0;
4357 }
4358
4359 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset)
4360 {
4361         struct amdgpu_device *adev = ring->adev;
4362         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4363         int mqd_idx = ring - &adev->gfx.compute_ring[0];
4364
4365         if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
4366                 memset((void *)mqd, 0, sizeof(*mqd));
4367                 mutex_lock(&adev->srbm_mutex);
4368                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4369                 amdgpu_ring_init_mqd(ring);
4370                 soc21_grbm_select(adev, 0, 0, 0, 0);
4371                 mutex_unlock(&adev->srbm_mutex);
4372
4373                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4374                         memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4375         } else {
4376                 /* restore MQD to a clean status */
4377                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4378                         memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4379                 /* reset ring buffer */
4380                 ring->wptr = 0;
4381                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4382                 amdgpu_ring_clear_ring(ring);
4383         }
4384
4385         return 0;
4386 }
4387
4388 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev)
4389 {
4390         struct amdgpu_ring *ring;
4391         int r;
4392
4393         ring = &adev->gfx.kiq[0].ring;
4394
4395         r = amdgpu_bo_reserve(ring->mqd_obj, false);
4396         if (unlikely(r != 0))
4397                 return r;
4398
4399         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4400         if (unlikely(r != 0)) {
4401                 amdgpu_bo_unreserve(ring->mqd_obj);
4402                 return r;
4403         }
4404
4405         gfx_v11_0_kiq_init_queue(ring);
4406         amdgpu_bo_kunmap(ring->mqd_obj);
4407         ring->mqd_ptr = NULL;
4408         amdgpu_bo_unreserve(ring->mqd_obj);
4409         ring->sched.ready = true;
4410         return 0;
4411 }
4412
4413 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)
4414 {
4415         struct amdgpu_ring *ring = NULL;
4416         int r = 0, i;
4417
4418         if (!amdgpu_async_gfx_ring)
4419                 gfx_v11_0_cp_compute_enable(adev, true);
4420
4421         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4422                 ring = &adev->gfx.compute_ring[i];
4423
4424                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4425                 if (unlikely(r != 0))
4426                         goto done;
4427                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4428                 if (!r) {
4429                         r = gfx_v11_0_kcq_init_queue(ring, false);
4430                         amdgpu_bo_kunmap(ring->mqd_obj);
4431                         ring->mqd_ptr = NULL;
4432                 }
4433                 amdgpu_bo_unreserve(ring->mqd_obj);
4434                 if (r)
4435                         goto done;
4436         }
4437
4438         r = amdgpu_gfx_enable_kcq(adev, 0);
4439 done:
4440         return r;
4441 }
4442
4443 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
4444 {
4445         int r, i;
4446         struct amdgpu_ring *ring;
4447
4448         if (!(adev->flags & AMD_IS_APU))
4449                 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4450
4451         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4452                 /* legacy firmware loading */
4453                 r = gfx_v11_0_cp_gfx_load_microcode(adev);
4454                 if (r)
4455                         return r;
4456
4457                 if (adev->gfx.rs64_enable)
4458                         r = gfx_v11_0_cp_compute_load_microcode_rs64(adev);
4459                 else
4460                         r = gfx_v11_0_cp_compute_load_microcode(adev);
4461                 if (r)
4462                         return r;
4463         }
4464
4465         gfx_v11_0_cp_set_doorbell_range(adev);
4466
4467         if (amdgpu_async_gfx_ring) {
4468                 gfx_v11_0_cp_compute_enable(adev, true);
4469                 gfx_v11_0_cp_gfx_enable(adev, true);
4470         }
4471
4472         if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
4473                 r = amdgpu_mes_kiq_hw_init(adev);
4474         else
4475                 r = gfx_v11_0_kiq_resume(adev);
4476         if (r)
4477                 return r;
4478
4479         r = gfx_v11_0_kcq_resume(adev);
4480         if (r)
4481                 return r;
4482
4483         if (!amdgpu_async_gfx_ring) {
4484                 r = gfx_v11_0_cp_gfx_resume(adev);
4485                 if (r)
4486                         return r;
4487         } else {
4488                 r = gfx_v11_0_cp_async_gfx_ring_resume(adev);
4489                 if (r)
4490                         return r;
4491         }
4492
4493         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4494                 ring = &adev->gfx.gfx_ring[i];
4495                 r = amdgpu_ring_test_helper(ring);
4496                 if (r)
4497                         return r;
4498         }
4499
4500         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4501                 ring = &adev->gfx.compute_ring[i];
4502                 r = amdgpu_ring_test_helper(ring);
4503                 if (r)
4504                         return r;
4505         }
4506
4507         return 0;
4508 }
4509
4510 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable)
4511 {
4512         gfx_v11_0_cp_gfx_enable(adev, enable);
4513         gfx_v11_0_cp_compute_enable(adev, enable);
4514 }
4515
4516 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev)
4517 {
4518         int r;
4519         bool value;
4520
4521         r = adev->gfxhub.funcs->gart_enable(adev);
4522         if (r)
4523                 return r;
4524
4525         adev->hdp.funcs->flush_hdp(adev, NULL);
4526
4527         value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
4528                 false : true;
4529
4530         adev->gfxhub.funcs->set_fault_enable_default(adev, value);
4531         /* TODO investigate why this and the hdp flush above is needed,
4532          * are we missing a flush somewhere else? */
4533         adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
4534
4535         return 0;
4536 }
4537
4538 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev)
4539 {
4540         u32 tmp;
4541
4542         /* select RS64 */
4543         if (adev->gfx.rs64_enable) {
4544                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL);
4545                 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1);
4546                 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp);
4547
4548                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL);
4549                 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1);
4550                 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp);
4551         }
4552
4553         if (amdgpu_emu_mode == 1)
4554                 msleep(100);
4555 }
4556
4557 static int get_gb_addr_config(struct amdgpu_device * adev)
4558 {
4559         u32 gb_addr_config;
4560
4561         gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
4562         if (gb_addr_config == 0)
4563                 return -EINVAL;
4564
4565         adev->gfx.config.gb_addr_config_fields.num_pkrs =
4566                 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
4567
4568         adev->gfx.config.gb_addr_config = gb_addr_config;
4569
4570         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
4571                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4572                                       GB_ADDR_CONFIG, NUM_PIPES);
4573
4574         adev->gfx.config.max_tile_pipes =
4575                 adev->gfx.config.gb_addr_config_fields.num_pipes;
4576
4577         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
4578                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4579                                       GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
4580         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
4581                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4582                                       GB_ADDR_CONFIG, NUM_RB_PER_SE);
4583         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
4584                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4585                                       GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
4586         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
4587                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4588                                       GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
4589
4590         return 0;
4591 }
4592
4593 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)
4594 {
4595         uint32_t data;
4596
4597         data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
4598         data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
4599         WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
4600
4601         data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
4602         data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
4603         WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
4604 }
4605
4606 static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
4607 {
4608         int r;
4609         struct amdgpu_device *adev = ip_block->adev;
4610
4611         amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
4612                                        adev->gfx.cleaner_shader_ptr);
4613
4614         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
4615                 if (adev->gfx.imu.funcs) {
4616                         /* RLC autoload sequence 1: Program rlc ram */
4617                         if (adev->gfx.imu.funcs->program_rlc_ram)
4618                                 adev->gfx.imu.funcs->program_rlc_ram(adev);
4619                         /* rlc autoload firmware */
4620                         r = gfx_v11_0_rlc_backdoor_autoload_enable(adev);
4621                         if (r)
4622                                 return r;
4623                 }
4624         } else {
4625                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4626                         if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
4627                                 if (adev->gfx.imu.funcs->load_microcode)
4628                                         adev->gfx.imu.funcs->load_microcode(adev);
4629                                 if (adev->gfx.imu.funcs->setup_imu)
4630                                         adev->gfx.imu.funcs->setup_imu(adev);
4631                                 if (adev->gfx.imu.funcs->start_imu)
4632                                         adev->gfx.imu.funcs->start_imu(adev);
4633                         }
4634
4635                         /* disable gpa mode in backdoor loading */
4636                         gfx_v11_0_disable_gpa_mode(adev);
4637                 }
4638         }
4639
4640         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
4641             (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
4642                 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev);
4643                 if (r) {
4644                         dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
4645                         return r;
4646                 }
4647         }
4648
4649         adev->gfx.is_poweron = true;
4650
4651         if(get_gb_addr_config(adev))
4652                 DRM_WARN("Invalid gb_addr_config !\n");
4653
4654         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
4655             adev->gfx.rs64_enable)
4656                 gfx_v11_0_config_gfx_rs64(adev);
4657
4658         r = gfx_v11_0_gfxhub_enable(adev);
4659         if (r)
4660                 return r;
4661
4662         if (!amdgpu_emu_mode)
4663                 gfx_v11_0_init_golden_registers(adev);
4664
4665         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
4666             (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) {
4667                 /**
4668                  * For gfx 11, rlc firmware loading relies on smu firmware is
4669                  * loaded firstly, so in direct type, it has to load smc ucode
4670                  * here before rlc.
4671                  */
4672                 r = amdgpu_pm_load_smu_firmware(adev, NULL);
4673                 if (r)
4674                         return r;
4675         }
4676
4677         gfx_v11_0_constants_init(adev);
4678
4679         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
4680                 gfx_v11_0_select_cp_fw_arch(adev);
4681
4682         if (adev->nbio.funcs->gc_doorbell_init)
4683                 adev->nbio.funcs->gc_doorbell_init(adev);
4684
4685         r = gfx_v11_0_rlc_resume(adev);
4686         if (r)
4687                 return r;
4688
4689         /*
4690          * init golden registers and rlc resume may override some registers,
4691          * reconfig them here
4692          */
4693         gfx_v11_0_tcp_harvest(adev);
4694
4695         r = gfx_v11_0_cp_resume(adev);
4696         if (r)
4697                 return r;
4698
4699         /* get IMU version from HW if it's not set */
4700         if (!adev->gfx.imu_fw_version)
4701                 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0);
4702
4703         return r;
4704 }
4705
4706 static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block)
4707 {
4708         struct amdgpu_device *adev = ip_block->adev;
4709
4710         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4711         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4712         amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
4713
4714         if (!adev->no_hw_access) {
4715                 if (amdgpu_async_gfx_ring) {
4716                         if (amdgpu_gfx_disable_kgq(adev, 0))
4717                                 DRM_ERROR("KGQ disable failed\n");
4718                 }
4719
4720                 if (amdgpu_gfx_disable_kcq(adev, 0))
4721                         DRM_ERROR("KCQ disable failed\n");
4722
4723                 amdgpu_mes_kiq_hw_fini(adev);
4724         }
4725
4726         if (amdgpu_sriov_vf(adev))
4727                 /* Remove the steps disabling CPG and clearing KIQ position,
4728                  * so that CP could perform IDLE-SAVE during switch. Those
4729                  * steps are necessary to avoid a DMAR error in gfx9 but it is
4730                  * not reproduced on gfx11.
4731                  */
4732                 return 0;
4733
4734         gfx_v11_0_cp_enable(adev, false);
4735         gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4736
4737         adev->gfxhub.funcs->gart_disable(adev);
4738
4739         adev->gfx.is_poweron = false;
4740
4741         return 0;
4742 }
4743
4744 static int gfx_v11_0_suspend(struct amdgpu_ip_block *ip_block)
4745 {
4746         return gfx_v11_0_hw_fini(ip_block);
4747 }
4748
4749 static int gfx_v11_0_resume(struct amdgpu_ip_block *ip_block)
4750 {
4751         return gfx_v11_0_hw_init(ip_block);
4752 }
4753
4754 static bool gfx_v11_0_is_idle(void *handle)
4755 {
4756         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4757
4758         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
4759                                 GRBM_STATUS, GUI_ACTIVE))
4760                 return false;
4761         else
4762                 return true;
4763 }
4764
4765 static int gfx_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
4766 {
4767         unsigned i;
4768         u32 tmp;
4769         struct amdgpu_device *adev = ip_block->adev;
4770
4771         for (i = 0; i < adev->usec_timeout; i++) {
4772                 /* read MC_STATUS */
4773                 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
4774                         GRBM_STATUS__GUI_ACTIVE_MASK;
4775
4776                 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
4777                         return 0;
4778                 udelay(1);
4779         }
4780         return -ETIMEDOUT;
4781 }
4782
4783 int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev,
4784                                       bool req)
4785 {
4786         u32 i, tmp, val;
4787
4788         for (i = 0; i < adev->usec_timeout; i++) {
4789                 /* Request with MeId=2, PipeId=0 */
4790                 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req);
4791                 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4);
4792                 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp);
4793
4794                 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX);
4795                 if (req) {
4796                         if (val == tmp)
4797                                 break;
4798                 } else {
4799                         tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX,
4800                                             REQUEST, 1);
4801
4802                         /* unlocked or locked by firmware */
4803                         if (val != tmp)
4804                                 break;
4805                 }
4806                 udelay(1);
4807         }
4808
4809         if (i >= adev->usec_timeout)
4810                 return -EINVAL;
4811
4812         return 0;
4813 }
4814
4815 static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block)
4816 {
4817         u32 grbm_soft_reset = 0;
4818         u32 tmp;
4819         int r, i, j, k;
4820         struct amdgpu_device *adev = ip_block->adev;
4821
4822         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4823
4824         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4825         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0);
4826         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0);
4827         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0);
4828         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0);
4829         WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4830
4831         mutex_lock(&adev->srbm_mutex);
4832         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
4833                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
4834                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
4835                                 soc21_grbm_select(adev, i, k, j, 0);
4836
4837                                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
4838                                 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
4839                         }
4840                 }
4841         }
4842         for (i = 0; i < adev->gfx.me.num_me; ++i) {
4843                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
4844                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
4845                                 soc21_grbm_select(adev, i, k, j, 0);
4846
4847                                 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1);
4848                         }
4849                 }
4850         }
4851         soc21_grbm_select(adev, 0, 0, 0, 0);
4852         mutex_unlock(&adev->srbm_mutex);
4853
4854         /* Try to acquire the gfx mutex before access to CP_VMID_RESET */
4855         mutex_lock(&adev->gfx.reset_sem_mutex);
4856         r = gfx_v11_0_request_gfx_index_mutex(adev, true);
4857         if (r) {
4858                 mutex_unlock(&adev->gfx.reset_sem_mutex);
4859                 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n");
4860                 return r;
4861         }
4862
4863         WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe);
4864
4865         // Read CP_VMID_RESET register three times.
4866         // to get sufficient time for GFX_HQD_ACTIVE reach 0
4867         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4868         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4869         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4870
4871         /* release the gfx mutex */
4872         r = gfx_v11_0_request_gfx_index_mutex(adev, false);
4873         mutex_unlock(&adev->gfx.reset_sem_mutex);
4874         if (r) {
4875                 DRM_ERROR("Failed to release the gfx mutex during soft reset\n");
4876                 return r;
4877         }
4878
4879         for (i = 0; i < adev->usec_timeout; i++) {
4880                 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) &&
4881                     !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE))
4882                         break;
4883                 udelay(1);
4884         }
4885         if (i >= adev->usec_timeout) {
4886                 printk("Failed to wait all pipes clean\n");
4887                 return -EINVAL;
4888         }
4889
4890         /**********  trigger soft reset  ***********/
4891         grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4892         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4893                                         SOFT_RESET_CP, 1);
4894         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4895                                         SOFT_RESET_GFX, 1);
4896         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4897                                         SOFT_RESET_CPF, 1);
4898         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4899                                         SOFT_RESET_CPC, 1);
4900         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4901                                         SOFT_RESET_CPG, 1);
4902         WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4903         /**********  exit soft reset  ***********/
4904         grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4905         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4906                                         SOFT_RESET_CP, 0);
4907         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4908                                         SOFT_RESET_GFX, 0);
4909         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4910                                         SOFT_RESET_CPF, 0);
4911         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4912                                         SOFT_RESET_CPC, 0);
4913         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4914                                         SOFT_RESET_CPG, 0);
4915         WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4916
4917         tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL);
4918         tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1);
4919         WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp);
4920
4921         WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0);
4922         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0);
4923
4924         for (i = 0; i < adev->usec_timeout; i++) {
4925                 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET))
4926                         break;
4927                 udelay(1);
4928         }
4929         if (i >= adev->usec_timeout) {
4930                 printk("Failed to wait CP_VMID_RESET to 0\n");
4931                 return -EINVAL;
4932         }
4933
4934         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4935         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
4936         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
4937         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
4938         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
4939         WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4940
4941         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4942
4943         return gfx_v11_0_cp_resume(adev);
4944 }
4945
4946 static bool gfx_v11_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
4947 {
4948         int i, r;
4949         struct amdgpu_device *adev = ip_block->adev;
4950         struct amdgpu_ring *ring;
4951         long tmo = msecs_to_jiffies(1000);
4952
4953         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4954                 ring = &adev->gfx.gfx_ring[i];
4955                 r = amdgpu_ring_test_ib(ring, tmo);
4956                 if (r)
4957                         return true;
4958         }
4959
4960         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4961                 ring = &adev->gfx.compute_ring[i];
4962                 r = amdgpu_ring_test_ib(ring, tmo);
4963                 if (r)
4964                         return true;
4965         }
4966
4967         return false;
4968 }
4969
4970 static int gfx_v11_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
4971 {
4972         struct amdgpu_device *adev = ip_block->adev;
4973         /**
4974          * GFX soft reset will impact MES, need resume MES when do GFX soft reset
4975          */
4976         return amdgpu_mes_resume(adev);
4977 }
4978
4979 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4980 {
4981         uint64_t clock;
4982         uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after;
4983
4984         if (amdgpu_sriov_vf(adev)) {
4985                 amdgpu_gfx_off_ctrl(adev, false);
4986                 mutex_lock(&adev->gfx.gpu_clock_mutex);
4987                 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
4988                 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
4989                 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
4990                 if (clock_counter_hi_pre != clock_counter_hi_after)
4991                         clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
4992                 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4993                 amdgpu_gfx_off_ctrl(adev, true);
4994         } else {
4995                 preempt_disable();
4996                 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
4997                 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
4998                 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
4999                 if (clock_counter_hi_pre != clock_counter_hi_after)
5000                         clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
5001                 preempt_enable();
5002         }
5003         clock = clock_counter_lo | (clock_counter_hi_after << 32ULL);
5004
5005         return clock;
5006 }
5007
5008 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
5009                                            uint32_t vmid,
5010                                            uint32_t gds_base, uint32_t gds_size,
5011                                            uint32_t gws_base, uint32_t gws_size,
5012                                            uint32_t oa_base, uint32_t oa_size)
5013 {
5014         struct amdgpu_device *adev = ring->adev;
5015
5016         /* GDS Base */
5017         gfx_v11_0_write_data_to_reg(ring, 0, false,
5018                                     SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid,
5019                                     gds_base);
5020
5021         /* GDS Size */
5022         gfx_v11_0_write_data_to_reg(ring, 0, false,
5023                                     SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid,
5024                                     gds_size);
5025
5026         /* GWS */
5027         gfx_v11_0_write_data_to_reg(ring, 0, false,
5028                                     SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid,
5029                                     gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
5030
5031         /* OA */
5032         gfx_v11_0_write_data_to_reg(ring, 0, false,
5033                                     SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid,
5034                                     (1 << (oa_size + oa_base)) - (1 << oa_base));
5035 }
5036
5037 static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
5038 {
5039         struct amdgpu_device *adev = ip_block->adev;
5040
5041         adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
5042
5043         adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
5044         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
5045                                           AMDGPU_MAX_COMPUTE_RINGS);
5046
5047         gfx_v11_0_set_kiq_pm4_funcs(adev);
5048         gfx_v11_0_set_ring_funcs(adev);
5049         gfx_v11_0_set_irq_funcs(adev);
5050         gfx_v11_0_set_gds_init(adev);
5051         gfx_v11_0_set_rlc_funcs(adev);
5052         gfx_v11_0_set_mqd_funcs(adev);
5053         gfx_v11_0_set_imu_funcs(adev);
5054
5055         gfx_v11_0_init_rlcg_reg_access_ctrl(adev);
5056
5057         return gfx_v11_0_init_microcode(adev);
5058 }
5059
5060 static int gfx_v11_0_late_init(struct amdgpu_ip_block *ip_block)
5061 {
5062         struct amdgpu_device *adev = ip_block->adev;
5063         int r;
5064
5065         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5066         if (r)
5067                 return r;
5068
5069         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5070         if (r)
5071                 return r;
5072
5073         r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
5074         if (r)
5075                 return r;
5076         return 0;
5077 }
5078
5079 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
5080 {
5081         uint32_t rlc_cntl;
5082
5083         /* if RLC is not enabled, do nothing */
5084         rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
5085         return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
5086 }
5087
5088 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
5089 {
5090         uint32_t data;
5091         unsigned i;
5092
5093         data = RLC_SAFE_MODE__CMD_MASK;
5094         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5095
5096         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
5097
5098         /* wait for RLC_SAFE_MODE */
5099         for (i = 0; i < adev->usec_timeout; i++) {
5100                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
5101                                    RLC_SAFE_MODE, CMD))
5102                         break;
5103                 udelay(1);
5104         }
5105 }
5106
5107 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
5108 {
5109         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
5110 }
5111
5112 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
5113                                       bool enable)
5114 {
5115         uint32_t def, data;
5116
5117         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
5118                 return;
5119
5120         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5121
5122         if (enable)
5123                 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5124         else
5125                 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5126
5127         if (def != data)
5128                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5129 }
5130
5131 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev,
5132                                        bool enable)
5133 {
5134         uint32_t def, data;
5135
5136         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
5137                 return;
5138
5139         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5140
5141         if (enable)
5142                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5143         else
5144                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5145
5146         if (def != data)
5147                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5148 }
5149
5150 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev,
5151                                            bool enable)
5152 {
5153         uint32_t def, data;
5154
5155         if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
5156                 return;
5157
5158         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5159
5160         if (enable)
5161                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5162         else
5163                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5164
5165         if (def != data)
5166                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5167 }
5168
5169 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5170                                                        bool enable)
5171 {
5172         uint32_t data, def;
5173
5174         if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
5175                 return;
5176
5177         /* It is disabled by HW by default */
5178         if (enable) {
5179                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5180                         /* 1 - RLC_CGTT_MGCG_OVERRIDE */
5181                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5182
5183                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5184                                   RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5185                                   RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5186
5187                         if (def != data)
5188                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5189                 }
5190         } else {
5191                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5192                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5193
5194                         data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5195                                  RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5196                                  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5197
5198                         if (def != data)
5199                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5200                 }
5201         }
5202 }
5203
5204 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5205                                                        bool enable)
5206 {
5207         uint32_t def, data;
5208
5209         if (!(adev->cg_flags &
5210               (AMD_CG_SUPPORT_GFX_CGCG |
5211               AMD_CG_SUPPORT_GFX_CGLS |
5212               AMD_CG_SUPPORT_GFX_3D_CGCG |
5213               AMD_CG_SUPPORT_GFX_3D_CGLS)))
5214                 return;
5215
5216         if (enable) {
5217                 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5218
5219                 /* unset CGCG override */
5220                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5221                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5222                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5223                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5224                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
5225                     adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5226                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5227
5228                 /* update CGCG override bits */
5229                 if (def != data)
5230                         WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5231
5232                 /* enable cgcg FSM(0x0000363F) */
5233                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5234
5235                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5236                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
5237                         data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5238                                  RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5239                 }
5240
5241                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5242                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
5243                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5244                                  RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5245                 }
5246
5247                 if (def != data)
5248                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5249
5250                 /* Program RLC_CGCG_CGLS_CTRL_3D */
5251                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5252
5253                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5254                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
5255                         data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5256                                  RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5257                 }
5258
5259                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5260                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
5261                         data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5262                                  RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5263                 }
5264
5265                 if (def != data)
5266                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5267
5268                 /* set IDLE_POLL_COUNT(0x00900100) */
5269                 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
5270
5271                 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
5272                 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5273                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5274
5275                 if (def != data)
5276                         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
5277
5278                 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5279                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
5280                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
5281                 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
5282                 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
5283                 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
5284
5285                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5286                 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5287                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5288
5289                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5290                 if (adev->sdma.num_instances > 1) {
5291                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5292                         data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5293                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5294                 }
5295         } else {
5296                 /* Program RLC_CGCG_CGLS_CTRL */
5297                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5298
5299                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5300                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5301
5302                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5303                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5304
5305                 if (def != data)
5306                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5307
5308                 /* Program RLC_CGCG_CGLS_CTRL_3D */
5309                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5310
5311                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5312                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5313                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5314                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5315
5316                 if (def != data)
5317                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5318
5319                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5320                 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5321                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5322
5323                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5324                 if (adev->sdma.num_instances > 1) {
5325                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5326                         data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5327                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5328                 }
5329         }
5330 }
5331
5332 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5333                                             bool enable)
5334 {
5335         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5336
5337         gfx_v11_0_update_coarse_grain_clock_gating(adev, enable);
5338
5339         gfx_v11_0_update_medium_grain_clock_gating(adev, enable);
5340
5341         gfx_v11_0_update_repeater_fgcg(adev, enable);
5342
5343         gfx_v11_0_update_sram_fgcg(adev, enable);
5344
5345         gfx_v11_0_update_perf_clk(adev, enable);
5346
5347         if (adev->cg_flags &
5348             (AMD_CG_SUPPORT_GFX_MGCG |
5349              AMD_CG_SUPPORT_GFX_CGLS |
5350              AMD_CG_SUPPORT_GFX_CGCG |
5351              AMD_CG_SUPPORT_GFX_3D_CGCG |
5352              AMD_CG_SUPPORT_GFX_3D_CGLS))
5353                 gfx_v11_0_enable_gui_idle_interrupt(adev, enable);
5354
5355         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5356
5357         return 0;
5358 }
5359
5360 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)
5361 {
5362         u32 reg, pre_data, data;
5363
5364         amdgpu_gfx_off_ctrl(adev, false);
5365         reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
5366         if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))
5367                 pre_data = RREG32_NO_KIQ(reg);
5368         else
5369                 pre_data = RREG32(reg);
5370
5371         data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK);
5372         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5373
5374         if (pre_data != data) {
5375                 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {
5376                         WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
5377                 } else
5378                         WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
5379         }
5380         amdgpu_gfx_off_ctrl(adev, true);
5381
5382         if (ring
5383                 && amdgpu_sriov_is_pp_one_vf(adev)
5384                 && (pre_data != data)
5385                 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
5386                         || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
5387                 amdgpu_ring_emit_wreg(ring, reg, data);
5388         }
5389 }
5390
5391 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
5392         .is_rlc_enabled = gfx_v11_0_is_rlc_enabled,
5393         .set_safe_mode = gfx_v11_0_set_safe_mode,
5394         .unset_safe_mode = gfx_v11_0_unset_safe_mode,
5395         .init = gfx_v11_0_rlc_init,
5396         .get_csb_size = gfx_v11_0_get_csb_size,
5397         .get_csb_buffer = gfx_v11_0_get_csb_buffer,
5398         .resume = gfx_v11_0_rlc_resume,
5399         .stop = gfx_v11_0_rlc_stop,
5400         .reset = gfx_v11_0_rlc_reset,
5401         .start = gfx_v11_0_rlc_start,
5402         .update_spm_vmid = gfx_v11_0_update_spm_vmid,
5403 };
5404
5405 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable)
5406 {
5407         u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
5408
5409         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))
5410                 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5411         else
5412                 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5413
5414         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data);
5415
5416         // Program RLC_PG_DELAY3 for CGPG hysteresis
5417         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
5418                 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5419                 case IP_VERSION(11, 0, 1):
5420                 case IP_VERSION(11, 0, 4):
5421                 case IP_VERSION(11, 5, 0):
5422                 case IP_VERSION(11, 5, 1):
5423                 case IP_VERSION(11, 5, 2):
5424                         WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1);
5425                         break;
5426                 default:
5427                         break;
5428                 }
5429         }
5430 }
5431
5432 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable)
5433 {
5434         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5435
5436         gfx_v11_cntl_power_gating(adev, enable);
5437
5438         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5439 }
5440
5441 static int gfx_v11_0_set_powergating_state(void *handle,
5442                                            enum amd_powergating_state state)
5443 {
5444         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5445         bool enable = (state == AMD_PG_STATE_GATE);
5446
5447         if (amdgpu_sriov_vf(adev))
5448                 return 0;
5449
5450         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5451         case IP_VERSION(11, 0, 0):
5452         case IP_VERSION(11, 0, 2):
5453         case IP_VERSION(11, 0, 3):
5454                 amdgpu_gfx_off_ctrl(adev, enable);
5455                 break;
5456         case IP_VERSION(11, 0, 1):
5457         case IP_VERSION(11, 0, 4):
5458         case IP_VERSION(11, 5, 0):
5459         case IP_VERSION(11, 5, 1):
5460         case IP_VERSION(11, 5, 2):
5461                 if (!enable)
5462                         amdgpu_gfx_off_ctrl(adev, false);
5463
5464                 gfx_v11_cntl_pg(adev, enable);
5465
5466                 if (enable)
5467                         amdgpu_gfx_off_ctrl(adev, true);
5468
5469                 break;
5470         default:
5471                 break;
5472         }
5473
5474         return 0;
5475 }
5476
5477 static int gfx_v11_0_set_clockgating_state(void *handle,
5478                                           enum amd_clockgating_state state)
5479 {
5480         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5481
5482         if (amdgpu_sriov_vf(adev))
5483                 return 0;
5484
5485         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5486         case IP_VERSION(11, 0, 0):
5487         case IP_VERSION(11, 0, 1):
5488         case IP_VERSION(11, 0, 2):
5489         case IP_VERSION(11, 0, 3):
5490         case IP_VERSION(11, 0, 4):
5491         case IP_VERSION(11, 5, 0):
5492         case IP_VERSION(11, 5, 1):
5493         case IP_VERSION(11, 5, 2):
5494                 gfx_v11_0_update_gfx_clock_gating(adev,
5495                                 state ==  AMD_CG_STATE_GATE);
5496                 break;
5497         default:
5498                 break;
5499         }
5500
5501         return 0;
5502 }
5503
5504 static void gfx_v11_0_get_clockgating_state(void *handle, u64 *flags)
5505 {
5506         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5507         int data;
5508
5509         /* AMD_CG_SUPPORT_GFX_MGCG */
5510         data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5511         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5512                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5513
5514         /* AMD_CG_SUPPORT_REPEATER_FGCG */
5515         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
5516                 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
5517
5518         /* AMD_CG_SUPPORT_GFX_FGCG */
5519         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
5520                 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
5521
5522         /* AMD_CG_SUPPORT_GFX_PERF_CLK */
5523         if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
5524                 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
5525
5526         /* AMD_CG_SUPPORT_GFX_CGCG */
5527         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5528         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5529                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5530
5531         /* AMD_CG_SUPPORT_GFX_CGLS */
5532         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5533                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5534
5535         /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5536         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5537         if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5538                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5539
5540         /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5541         if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5542                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5543 }
5544
5545 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5546 {
5547         /* gfx11 is 32bit rptr*/
5548         return *(uint32_t *)ring->rptr_cpu_addr;
5549 }
5550
5551 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5552 {
5553         struct amdgpu_device *adev = ring->adev;
5554         u64 wptr;
5555
5556         /* XXX check if swapping is necessary on BE */
5557         if (ring->use_doorbell) {
5558                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5559         } else {
5560                 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
5561                 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
5562         }
5563
5564         return wptr;
5565 }
5566
5567 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5568 {
5569         struct amdgpu_device *adev = ring->adev;
5570
5571         if (ring->use_doorbell) {
5572                 /* XXX check if swapping is necessary on BE */
5573                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5574                              ring->wptr);
5575                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5576         } else {
5577                 WREG32_SOC15(GC, 0, regCP_RB0_WPTR,
5578                              lower_32_bits(ring->wptr));
5579                 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI,
5580                              upper_32_bits(ring->wptr));
5581         }
5582 }
5583
5584 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5585 {
5586         /* gfx11 hardware is 32bit rptr */
5587         return *(uint32_t *)ring->rptr_cpu_addr;
5588 }
5589
5590 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5591 {
5592         u64 wptr;
5593
5594         /* XXX check if swapping is necessary on BE */
5595         if (ring->use_doorbell)
5596                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5597         else
5598                 BUG();
5599         return wptr;
5600 }
5601
5602 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5603 {
5604         struct amdgpu_device *adev = ring->adev;
5605
5606         /* XXX check if swapping is necessary on BE */
5607         if (ring->use_doorbell) {
5608                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5609                              ring->wptr);
5610                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5611         } else {
5612                 BUG(); /* only DOORBELL method supported on gfx11 now */
5613         }
5614 }
5615
5616 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5617 {
5618         struct amdgpu_device *adev = ring->adev;
5619         u32 ref_and_mask, reg_mem_engine;
5620         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5621
5622         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5623                 switch (ring->me) {
5624                 case 1:
5625                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5626                         break;
5627                 case 2:
5628                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5629                         break;
5630                 default:
5631                         return;
5632                 }
5633                 reg_mem_engine = 0;
5634         } else {
5635                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe;
5636                 reg_mem_engine = 1; /* pfp */
5637         }
5638
5639         gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5640                                adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5641                                adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5642                                ref_and_mask, ref_and_mask, 0x20);
5643 }
5644
5645 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5646                                        struct amdgpu_job *job,
5647                                        struct amdgpu_ib *ib,
5648                                        uint32_t flags)
5649 {
5650         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5651         u32 header, control = 0;
5652
5653         BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE);
5654
5655         header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5656
5657         control |= ib->length_dw | (vmid << 24);
5658
5659         if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5660                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5661
5662                 if (flags & AMDGPU_IB_PREEMPTED)
5663                         control |= INDIRECT_BUFFER_PRE_RESUME(1);
5664
5665                 if (vmid)
5666                         gfx_v11_0_ring_emit_de_meta(ring,
5667                                     (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
5668         }
5669
5670         if (ring->is_mes_queue)
5671                 /* inherit vmid from mqd */
5672                 control |= 0x400000;
5673
5674         amdgpu_ring_write(ring, header);
5675         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5676         amdgpu_ring_write(ring,
5677 #ifdef __BIG_ENDIAN
5678                 (2 << 0) |
5679 #endif
5680                 lower_32_bits(ib->gpu_addr));
5681         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5682         amdgpu_ring_write(ring, control);
5683 }
5684
5685 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5686                                            struct amdgpu_job *job,
5687                                            struct amdgpu_ib *ib,
5688                                            uint32_t flags)
5689 {
5690         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5691         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5692
5693         if (ring->is_mes_queue)
5694                 /* inherit vmid from mqd */
5695                 control |= 0x40000000;
5696
5697         /* Currently, there is a high possibility to get wave ID mismatch
5698          * between ME and GDS, leading to a hw deadlock, because ME generates
5699          * different wave IDs than the GDS expects. This situation happens
5700          * randomly when at least 5 compute pipes use GDS ordered append.
5701          * The wave IDs generated by ME are also wrong after suspend/resume.
5702          * Those are probably bugs somewhere else in the kernel driver.
5703          *
5704          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5705          * GDS to 0 for this ring (me/pipe).
5706          */
5707         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5708                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5709                 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
5710                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5711         }
5712
5713         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5714         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5715         amdgpu_ring_write(ring,
5716 #ifdef __BIG_ENDIAN
5717                                 (2 << 0) |
5718 #endif
5719                                 lower_32_bits(ib->gpu_addr));
5720         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5721         amdgpu_ring_write(ring, control);
5722 }
5723
5724 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5725                                      u64 seq, unsigned flags)
5726 {
5727         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5728         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5729
5730         /* RELEASE_MEM - flush caches, send int */
5731         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5732         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
5733                                  PACKET3_RELEASE_MEM_GCR_GL2_WB |
5734                                  PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */
5735                                  PACKET3_RELEASE_MEM_GCR_GLM_WB |
5736                                  PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
5737                                  PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5738                                  PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
5739         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
5740                                  PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
5741
5742         /*
5743          * the address should be Qword aligned if 64bit write, Dword
5744          * aligned if only send 32bit data low (discard data high)
5745          */
5746         if (write64bit)
5747                 BUG_ON(addr & 0x7);
5748         else
5749                 BUG_ON(addr & 0x3);
5750         amdgpu_ring_write(ring, lower_32_bits(addr));
5751         amdgpu_ring_write(ring, upper_32_bits(addr));
5752         amdgpu_ring_write(ring, lower_32_bits(seq));
5753         amdgpu_ring_write(ring, upper_32_bits(seq));
5754         amdgpu_ring_write(ring, ring->is_mes_queue ?
5755                          (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0);
5756 }
5757
5758 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5759 {
5760         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5761         uint32_t seq = ring->fence_drv.sync_seq;
5762         uint64_t addr = ring->fence_drv.gpu_addr;
5763
5764         gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
5765                                upper_32_bits(addr), seq, 0xffffffff, 4);
5766 }
5767
5768 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
5769                                    uint16_t pasid, uint32_t flush_type,
5770                                    bool all_hub, uint8_t dst_sel)
5771 {
5772         amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
5773         amdgpu_ring_write(ring,
5774                           PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
5775                           PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
5776                           PACKET3_INVALIDATE_TLBS_PASID(pasid) |
5777                           PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
5778 }
5779
5780 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5781                                          unsigned vmid, uint64_t pd_addr)
5782 {
5783         if (ring->is_mes_queue)
5784                 gfx_v11_0_ring_invalidate_tlbs(ring, 0, 0, false, 0);
5785         else
5786                 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5787
5788         /* compute doesn't have PFP */
5789         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5790                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5791                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5792                 amdgpu_ring_write(ring, 0x0);
5793         }
5794
5795         /* Make sure that we can't skip the SET_Q_MODE packets when the VM
5796          * changed in any way.
5797          */
5798         ring->set_q_mode_offs = 0;
5799         ring->set_q_mode_ptr = NULL;
5800 }
5801
5802 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5803                                           u64 seq, unsigned int flags)
5804 {
5805         struct amdgpu_device *adev = ring->adev;
5806
5807         /* we only allocate 32bit for each seq wb address */
5808         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5809
5810         /* write fence seq to the "addr" */
5811         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5812         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5813                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5814         amdgpu_ring_write(ring, lower_32_bits(addr));
5815         amdgpu_ring_write(ring, upper_32_bits(addr));
5816         amdgpu_ring_write(ring, lower_32_bits(seq));
5817
5818         if (flags & AMDGPU_FENCE_FLAG_INT) {
5819                 /* set register to trigger INT */
5820                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5821                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5822                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5823                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
5824                 amdgpu_ring_write(ring, 0);
5825                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5826         }
5827 }
5828
5829 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
5830                                          uint32_t flags)
5831 {
5832         uint32_t dw2 = 0;
5833
5834         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5835         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5836                 /* set load_global_config & load_global_uconfig */
5837                 dw2 |= 0x8001;
5838                 /* set load_cs_sh_regs */
5839                 dw2 |= 0x01000000;
5840                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5841                 dw2 |= 0x10002;
5842         }
5843
5844         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5845         amdgpu_ring_write(ring, dw2);
5846         amdgpu_ring_write(ring, 0);
5847 }
5848
5849 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5850                                                    uint64_t addr)
5851 {
5852         unsigned ret;
5853
5854         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5855         amdgpu_ring_write(ring, lower_32_bits(addr));
5856         amdgpu_ring_write(ring, upper_32_bits(addr));
5857         /* discard following DWs if *cond_exec_gpu_addr==0 */
5858         amdgpu_ring_write(ring, 0);
5859         ret = ring->wptr & ring->buf_mask;
5860         /* patch dummy value later */
5861         amdgpu_ring_write(ring, 0);
5862
5863         return ret;
5864 }
5865
5866 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring,
5867                                            u64 shadow_va, u64 csa_va,
5868                                            u64 gds_va, bool init_shadow,
5869                                            int vmid)
5870 {
5871         struct amdgpu_device *adev = ring->adev;
5872         unsigned int offs, end;
5873
5874         if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj)
5875                 return;
5876
5877         /*
5878          * The logic here isn't easy to understand because we need to keep state
5879          * accross multiple executions of the function as well as between the
5880          * CPU and GPU. The general idea is that the newly written GPU command
5881          * has a condition on the previous one and only executed if really
5882          * necessary.
5883          */
5884
5885         /*
5886          * The dw in the NOP controls if the next SET_Q_MODE packet should be
5887          * executed or not. Reserve 64bits just to be on the save side.
5888          */
5889         amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1));
5890         offs = ring->wptr & ring->buf_mask;
5891
5892         /*
5893          * We start with skipping the prefix SET_Q_MODE and always executing
5894          * the postfix SET_Q_MODE packet. This is changed below with a
5895          * WRITE_DATA command when the postfix executed.
5896          */
5897         amdgpu_ring_write(ring, shadow_va ? 1 : 0);
5898         amdgpu_ring_write(ring, 0);
5899
5900         if (ring->set_q_mode_offs) {
5901                 uint64_t addr;
5902
5903                 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
5904                 addr += ring->set_q_mode_offs << 2;
5905                 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr);
5906         }
5907
5908         /*
5909          * When the postfix SET_Q_MODE packet executes we need to make sure that the
5910          * next prefix SET_Q_MODE packet executes as well.
5911          */
5912         if (!shadow_va) {
5913                 uint64_t addr;
5914
5915                 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
5916                 addr += offs << 2;
5917                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5918                 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM);
5919                 amdgpu_ring_write(ring, lower_32_bits(addr));
5920                 amdgpu_ring_write(ring, upper_32_bits(addr));
5921                 amdgpu_ring_write(ring, 0x1);
5922         }
5923
5924         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7));
5925         amdgpu_ring_write(ring, lower_32_bits(shadow_va));
5926         amdgpu_ring_write(ring, upper_32_bits(shadow_va));
5927         amdgpu_ring_write(ring, lower_32_bits(gds_va));
5928         amdgpu_ring_write(ring, upper_32_bits(gds_va));
5929         amdgpu_ring_write(ring, lower_32_bits(csa_va));
5930         amdgpu_ring_write(ring, upper_32_bits(csa_va));
5931         amdgpu_ring_write(ring, shadow_va ?
5932                           PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0);
5933         amdgpu_ring_write(ring, init_shadow ?
5934                           PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0);
5935
5936         if (ring->set_q_mode_offs)
5937                 amdgpu_ring_patch_cond_exec(ring, end);
5938
5939         if (shadow_va) {
5940                 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid;
5941
5942                 /*
5943                  * If the tokens match try to skip the last postfix SET_Q_MODE
5944                  * packet to avoid saving/restoring the state all the time.
5945                  */
5946                 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token)
5947                         *ring->set_q_mode_ptr = 0;
5948
5949                 ring->set_q_mode_token = token;
5950         } else {
5951                 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs];
5952         }
5953
5954         ring->set_q_mode_offs = offs;
5955 }
5956
5957 static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)
5958 {
5959         int i, r = 0;
5960         struct amdgpu_device *adev = ring->adev;
5961         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5962         struct amdgpu_ring *kiq_ring = &kiq->ring;
5963         unsigned long flags;
5964
5965         if (adev->enable_mes)
5966                 return -EINVAL;
5967
5968         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5969                 return -EINVAL;
5970
5971         spin_lock_irqsave(&kiq->ring_lock, flags);
5972
5973         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5974                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5975                 return -ENOMEM;
5976         }
5977
5978         /* assert preemption condition */
5979         amdgpu_ring_set_preempt_cond_exec(ring, false);
5980
5981         /* assert IB preemption, emit the trailing fence */
5982         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5983                                    ring->trail_fence_gpu_addr,
5984                                    ++ring->trail_seq);
5985         amdgpu_ring_commit(kiq_ring);
5986
5987         spin_unlock_irqrestore(&kiq->ring_lock, flags);
5988
5989         /* poll the trailing fence */
5990         for (i = 0; i < adev->usec_timeout; i++) {
5991                 if (ring->trail_seq ==
5992                     le32_to_cpu(*(ring->trail_fence_cpu_addr)))
5993                         break;
5994                 udelay(1);
5995         }
5996
5997         if (i >= adev->usec_timeout) {
5998                 r = -EINVAL;
5999                 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
6000         }
6001
6002         /* deassert preemption condition */
6003         amdgpu_ring_set_preempt_cond_exec(ring, true);
6004         return r;
6005 }
6006
6007 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
6008 {
6009         struct amdgpu_device *adev = ring->adev;
6010         struct v10_de_ib_state de_payload = {0};
6011         uint64_t offset, gds_addr, de_payload_gpu_addr;
6012         void *de_payload_cpu_addr;
6013         int cnt;
6014
6015         if (ring->is_mes_queue) {
6016                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
6017                                   gfx[0].gfx_meta_data) +
6018                         offsetof(struct v10_gfx_meta_data, de_payload);
6019                 de_payload_gpu_addr =
6020                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
6021                 de_payload_cpu_addr =
6022                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
6023
6024                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
6025                                   gfx[0].gds_backup) +
6026                         offsetof(struct v10_gfx_meta_data, de_payload);
6027                 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
6028         } else {
6029                 offset = offsetof(struct v10_gfx_meta_data, de_payload);
6030                 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
6031                 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
6032
6033                 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
6034                                  AMDGPU_CSA_SIZE - adev->gds.gds_size,
6035                                  PAGE_SIZE);
6036         }
6037
6038         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
6039         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
6040
6041         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
6042         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
6043         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
6044                                  WRITE_DATA_DST_SEL(8) |
6045                                  WR_CONFIRM) |
6046                                  WRITE_DATA_CACHE_POLICY(0));
6047         amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
6048         amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
6049
6050         if (resume)
6051                 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
6052                                            sizeof(de_payload) >> 2);
6053         else
6054                 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
6055                                            sizeof(de_payload) >> 2);
6056 }
6057
6058 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
6059                                     bool secure)
6060 {
6061         uint32_t v = secure ? FRAME_TMZ : 0;
6062
6063         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
6064         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
6065 }
6066
6067 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
6068                                      uint32_t reg_val_offs)
6069 {
6070         struct amdgpu_device *adev = ring->adev;
6071
6072         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6073         amdgpu_ring_write(ring, 0 |     /* src: register*/
6074                                 (5 << 8) |      /* dst: memory */
6075                                 (1 << 20));     /* write confirm */
6076         amdgpu_ring_write(ring, reg);
6077         amdgpu_ring_write(ring, 0);
6078         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6079                                 reg_val_offs * 4));
6080         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6081                                 reg_val_offs * 4));
6082 }
6083
6084 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6085                                    uint32_t val)
6086 {
6087         uint32_t cmd = 0;
6088
6089         switch (ring->funcs->type) {
6090         case AMDGPU_RING_TYPE_GFX:
6091                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
6092                 break;
6093         case AMDGPU_RING_TYPE_KIQ:
6094                 cmd = (1 << 16); /* no inc addr */
6095                 break;
6096         default:
6097                 cmd = WR_CONFIRM;
6098                 break;
6099         }
6100         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6101         amdgpu_ring_write(ring, cmd);
6102         amdgpu_ring_write(ring, reg);
6103         amdgpu_ring_write(ring, 0);
6104         amdgpu_ring_write(ring, val);
6105 }
6106
6107 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
6108                                         uint32_t val, uint32_t mask)
6109 {
6110         gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
6111 }
6112
6113 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
6114                                                    uint32_t reg0, uint32_t reg1,
6115                                                    uint32_t ref, uint32_t mask)
6116 {
6117         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6118
6119         gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
6120                                ref, mask, 0x20);
6121 }
6122
6123 static void gfx_v11_0_ring_soft_recovery(struct amdgpu_ring *ring,
6124                                          unsigned vmid)
6125 {
6126         struct amdgpu_device *adev = ring->adev;
6127         uint32_t value = 0;
6128
6129         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
6130         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
6131         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
6132         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
6133         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
6134         WREG32_SOC15(GC, 0, regSQ_CMD, value);
6135         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
6136 }
6137
6138 static void
6139 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6140                                       uint32_t me, uint32_t pipe,
6141                                       enum amdgpu_interrupt_state state)
6142 {
6143         uint32_t cp_int_cntl, cp_int_cntl_reg;
6144
6145         if (!me) {
6146                 switch (pipe) {
6147                 case 0:
6148                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
6149                         break;
6150                 case 1:
6151                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
6152                         break;
6153                 default:
6154                         DRM_DEBUG("invalid pipe %d\n", pipe);
6155                         return;
6156                 }
6157         } else {
6158                 DRM_DEBUG("invalid me %d\n", me);
6159                 return;
6160         }
6161
6162         switch (state) {
6163         case AMDGPU_IRQ_STATE_DISABLE:
6164                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6165                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6166                                             TIME_STAMP_INT_ENABLE, 0);
6167                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6168                                             GENERIC0_INT_ENABLE, 0);
6169                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6170                 break;
6171         case AMDGPU_IRQ_STATE_ENABLE:
6172                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6173                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6174                                             TIME_STAMP_INT_ENABLE, 1);
6175                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6176                                             GENERIC0_INT_ENABLE, 1);
6177                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6178                 break;
6179         default:
6180                 break;
6181         }
6182 }
6183
6184 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6185                                                      int me, int pipe,
6186                                                      enum amdgpu_interrupt_state state)
6187 {
6188         u32 mec_int_cntl, mec_int_cntl_reg;
6189
6190         /*
6191          * amdgpu controls only the first MEC. That's why this function only
6192          * handles the setting of interrupts for this specific MEC. All other
6193          * pipes' interrupts are set by amdkfd.
6194          */
6195
6196         if (me == 1) {
6197                 switch (pipe) {
6198                 case 0:
6199                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6200                         break;
6201                 case 1:
6202                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
6203                         break;
6204                 case 2:
6205                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
6206                         break;
6207                 case 3:
6208                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
6209                         break;
6210                 default:
6211                         DRM_DEBUG("invalid pipe %d\n", pipe);
6212                         return;
6213                 }
6214         } else {
6215                 DRM_DEBUG("invalid me %d\n", me);
6216                 return;
6217         }
6218
6219         switch (state) {
6220         case AMDGPU_IRQ_STATE_DISABLE:
6221                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6222                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6223                                              TIME_STAMP_INT_ENABLE, 0);
6224                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6225                                              GENERIC0_INT_ENABLE, 0);
6226                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6227                 break;
6228         case AMDGPU_IRQ_STATE_ENABLE:
6229                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6230                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6231                                              TIME_STAMP_INT_ENABLE, 1);
6232                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6233                                              GENERIC0_INT_ENABLE, 1);
6234                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6235                 break;
6236         default:
6237                 break;
6238         }
6239 }
6240
6241 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6242                                             struct amdgpu_irq_src *src,
6243                                             unsigned type,
6244                                             enum amdgpu_interrupt_state state)
6245 {
6246         switch (type) {
6247         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6248                 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
6249                 break;
6250         case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
6251                 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
6252                 break;
6253         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6254                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6255                 break;
6256         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6257                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6258                 break;
6259         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6260                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6261                 break;
6262         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6263                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6264                 break;
6265         default:
6266                 break;
6267         }
6268         return 0;
6269 }
6270
6271 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
6272                              struct amdgpu_irq_src *source,
6273                              struct amdgpu_iv_entry *entry)
6274 {
6275         int i;
6276         u8 me_id, pipe_id, queue_id;
6277         struct amdgpu_ring *ring;
6278         uint32_t mes_queue_id = entry->src_data[0];
6279
6280         DRM_DEBUG("IH: CP EOP\n");
6281
6282         if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
6283                 struct amdgpu_mes_queue *queue;
6284
6285                 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
6286
6287                 spin_lock(&adev->mes.queue_id_lock);
6288                 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
6289                 if (queue) {
6290                         DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
6291                         amdgpu_fence_process(queue->ring);
6292                 }
6293                 spin_unlock(&adev->mes.queue_id_lock);
6294         } else {
6295                 me_id = (entry->ring_id & 0x0c) >> 2;
6296                 pipe_id = (entry->ring_id & 0x03) >> 0;
6297                 queue_id = (entry->ring_id & 0x70) >> 4;
6298
6299                 switch (me_id) {
6300                 case 0:
6301                         if (pipe_id == 0)
6302                                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6303                         else
6304                                 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
6305                         break;
6306                 case 1:
6307                 case 2:
6308                         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6309                                 ring = &adev->gfx.compute_ring[i];
6310                                 /* Per-queue interrupt is supported for MEC starting from VI.
6311                                  * The interrupt can only be enabled/disabled per pipe instead
6312                                  * of per queue.
6313                                  */
6314                                 if ((ring->me == me_id) &&
6315                                     (ring->pipe == pipe_id) &&
6316                                     (ring->queue == queue_id))
6317                                         amdgpu_fence_process(ring);
6318                         }
6319                         break;
6320                 }
6321         }
6322
6323         return 0;
6324 }
6325
6326 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6327                                               struct amdgpu_irq_src *source,
6328                                               unsigned int type,
6329                                               enum amdgpu_interrupt_state state)
6330 {
6331         u32 cp_int_cntl_reg, cp_int_cntl;
6332         int i, j;
6333
6334         switch (state) {
6335         case AMDGPU_IRQ_STATE_DISABLE:
6336         case AMDGPU_IRQ_STATE_ENABLE:
6337                 for (i = 0; i < adev->gfx.me.num_me; i++) {
6338                         for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6339                                 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6340
6341                                 if (cp_int_cntl_reg) {
6342                                         cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6343                                         cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6344                                                                     PRIV_REG_INT_ENABLE,
6345                                                                     state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6346                                         WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6347                                 }
6348                         }
6349                 }
6350                 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6351                         for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6352                                 /* MECs start at 1 */
6353                                 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j);
6354
6355                                 if (cp_int_cntl_reg) {
6356                                         cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6357                                         cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6358                                                                     PRIV_REG_INT_ENABLE,
6359                                                                     state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6360                                         WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6361                                 }
6362                         }
6363                 }
6364                 break;
6365         default:
6366                 break;
6367         }
6368
6369         return 0;
6370 }
6371
6372 static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6373                                             struct amdgpu_irq_src *source,
6374                                             unsigned type,
6375                                             enum amdgpu_interrupt_state state)
6376 {
6377         u32 cp_int_cntl_reg, cp_int_cntl;
6378         int i, j;
6379
6380         switch (state) {
6381         case AMDGPU_IRQ_STATE_DISABLE:
6382         case AMDGPU_IRQ_STATE_ENABLE:
6383                 for (i = 0; i < adev->gfx.me.num_me; i++) {
6384                         for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6385                                 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6386
6387                                 if (cp_int_cntl_reg) {
6388                                         cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6389                                         cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6390                                                                     OPCODE_ERROR_INT_ENABLE,
6391                                                                     state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6392                                         WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6393                                 }
6394                         }
6395                 }
6396                 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6397                         for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6398                                 /* MECs start at 1 */
6399                                 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j);
6400
6401                                 if (cp_int_cntl_reg) {
6402                                         cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6403                                         cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6404                                                                     OPCODE_ERROR_INT_ENABLE,
6405                                                                     state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6406                                         WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6407                                 }
6408                         }
6409                 }
6410                 break;
6411         default:
6412                 break;
6413         }
6414         return 0;
6415 }
6416
6417 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6418                                                struct amdgpu_irq_src *source,
6419                                                unsigned int type,
6420                                                enum amdgpu_interrupt_state state)
6421 {
6422         u32 cp_int_cntl_reg, cp_int_cntl;
6423         int i, j;
6424
6425         switch (state) {
6426         case AMDGPU_IRQ_STATE_DISABLE:
6427         case AMDGPU_IRQ_STATE_ENABLE:
6428                 for (i = 0; i < adev->gfx.me.num_me; i++) {
6429                         for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6430                                 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6431
6432                                 if (cp_int_cntl_reg) {
6433                                         cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6434                                         cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6435                                                                     PRIV_INSTR_INT_ENABLE,
6436                                                                     state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6437                                         WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6438                                 }
6439                         }
6440                 }
6441                 break;
6442         default:
6443                 break;
6444         }
6445
6446         return 0;
6447 }
6448
6449 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
6450                                         struct amdgpu_iv_entry *entry)
6451 {
6452         u8 me_id, pipe_id, queue_id;
6453         struct amdgpu_ring *ring;
6454         int i;
6455
6456         me_id = (entry->ring_id & 0x0c) >> 2;
6457         pipe_id = (entry->ring_id & 0x03) >> 0;
6458         queue_id = (entry->ring_id & 0x70) >> 4;
6459
6460         switch (me_id) {
6461         case 0:
6462                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
6463                         ring = &adev->gfx.gfx_ring[i];
6464                         if (ring->me == me_id && ring->pipe == pipe_id &&
6465                             ring->queue == queue_id)
6466                                 drm_sched_fault(&ring->sched);
6467                 }
6468                 break;
6469         case 1:
6470         case 2:
6471                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6472                         ring = &adev->gfx.compute_ring[i];
6473                         if (ring->me == me_id && ring->pipe == pipe_id &&
6474                             ring->queue == queue_id)
6475                                 drm_sched_fault(&ring->sched);
6476                 }
6477                 break;
6478         default:
6479                 BUG();
6480                 break;
6481         }
6482 }
6483
6484 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
6485                                   struct amdgpu_irq_src *source,
6486                                   struct amdgpu_iv_entry *entry)
6487 {
6488         DRM_ERROR("Illegal register access in command stream\n");
6489         gfx_v11_0_handle_priv_fault(adev, entry);
6490         return 0;
6491 }
6492
6493 static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev,
6494                                 struct amdgpu_irq_src *source,
6495                                 struct amdgpu_iv_entry *entry)
6496 {
6497         DRM_ERROR("Illegal opcode in command stream \n");
6498         gfx_v11_0_handle_priv_fault(adev, entry);
6499         return 0;
6500 }
6501
6502 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev,
6503                                    struct amdgpu_irq_src *source,
6504                                    struct amdgpu_iv_entry *entry)
6505 {
6506         DRM_ERROR("Illegal instruction in command stream\n");
6507         gfx_v11_0_handle_priv_fault(adev, entry);
6508         return 0;
6509 }
6510
6511 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev,
6512                                   struct amdgpu_irq_src *source,
6513                                   struct amdgpu_iv_entry *entry)
6514 {
6515         if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq)
6516                 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry);
6517
6518         return 0;
6519 }
6520
6521 #if 0
6522 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6523                                              struct amdgpu_irq_src *src,
6524                                              unsigned int type,
6525                                              enum amdgpu_interrupt_state state)
6526 {
6527         uint32_t tmp, target;
6528         struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring);
6529
6530         target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6531         target += ring->pipe;
6532
6533         switch (type) {
6534         case AMDGPU_CP_KIQ_IRQ_DRIVER0:
6535                 if (state == AMDGPU_IRQ_STATE_DISABLE) {
6536                         tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6537                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6538                                             GENERIC2_INT_ENABLE, 0);
6539                         WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6540
6541                         tmp = RREG32_SOC15_IP(GC, target);
6542                         tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6543                                             GENERIC2_INT_ENABLE, 0);
6544                         WREG32_SOC15_IP(GC, target, tmp);
6545                 } else {
6546                         tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6547                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6548                                             GENERIC2_INT_ENABLE, 1);
6549                         WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6550
6551                         tmp = RREG32_SOC15_IP(GC, target);
6552                         tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6553                                             GENERIC2_INT_ENABLE, 1);
6554                         WREG32_SOC15_IP(GC, target, tmp);
6555                 }
6556                 break;
6557         default:
6558                 BUG(); /* kiq only support GENERIC2_INT now */
6559                 break;
6560         }
6561         return 0;
6562 }
6563 #endif
6564
6565 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)
6566 {
6567         const unsigned int gcr_cntl =
6568                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
6569                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
6570                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
6571                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
6572                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
6573                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
6574                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
6575                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
6576
6577         /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
6578         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
6579         amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
6580         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6581         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6582         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6583         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6584         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6585         amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
6586 }
6587
6588 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid)
6589 {
6590         struct amdgpu_device *adev = ring->adev;
6591         int r;
6592
6593         if (amdgpu_sriov_vf(adev))
6594                 return -EINVAL;
6595
6596         r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false);
6597         if (r)
6598                 return r;
6599
6600         r = amdgpu_bo_reserve(ring->mqd_obj, false);
6601         if (unlikely(r != 0)) {
6602                 dev_err(adev->dev, "fail to resv mqd_obj\n");
6603                 return r;
6604         }
6605         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
6606         if (!r) {
6607                 r = gfx_v11_0_kgq_init_queue(ring, true);
6608                 amdgpu_bo_kunmap(ring->mqd_obj);
6609                 ring->mqd_ptr = NULL;
6610         }
6611         amdgpu_bo_unreserve(ring->mqd_obj);
6612         if (r) {
6613                 dev_err(adev->dev, "fail to unresv mqd_obj\n");
6614                 return r;
6615         }
6616
6617         r = amdgpu_mes_map_legacy_queue(adev, ring);
6618         if (r) {
6619                 dev_err(adev->dev, "failed to remap kgq\n");
6620                 return r;
6621         }
6622
6623         return amdgpu_ring_test_ring(ring);
6624 }
6625
6626 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring, unsigned int vmid)
6627 {
6628         struct amdgpu_device *adev = ring->adev;
6629         int i, r = 0;
6630
6631         if (amdgpu_sriov_vf(adev))
6632                 return -EINVAL;
6633
6634         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
6635         mutex_lock(&adev->srbm_mutex);
6636         soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
6637         WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
6638         WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
6639
6640         /* make sure dequeue is complete*/
6641         for (i = 0; i < adev->usec_timeout; i++) {
6642                 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
6643                         break;
6644                 udelay(1);
6645         }
6646         if (i >= adev->usec_timeout)
6647                 r = -ETIMEDOUT;
6648         soc21_grbm_select(adev, 0, 0, 0, 0);
6649         mutex_unlock(&adev->srbm_mutex);
6650         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
6651         if (r) {
6652                 dev_err(adev->dev, "fail to wait on hqd deactivate\n");
6653                 return r;
6654         }
6655
6656         r = amdgpu_bo_reserve(ring->mqd_obj, false);
6657         if (unlikely(r != 0)) {
6658                 dev_err(adev->dev, "fail to resv mqd_obj\n");
6659                 return r;
6660         }
6661         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
6662         if (!r) {
6663                 r = gfx_v11_0_kcq_init_queue(ring, true);
6664                 amdgpu_bo_kunmap(ring->mqd_obj);
6665                 ring->mqd_ptr = NULL;
6666         }
6667         amdgpu_bo_unreserve(ring->mqd_obj);
6668         if (r) {
6669                 dev_err(adev->dev, "fail to unresv mqd_obj\n");
6670                 return r;
6671         }
6672         r = amdgpu_mes_map_legacy_queue(adev, ring);
6673         if (r) {
6674                 dev_err(adev->dev, "failed to remap kcq\n");
6675                 return r;
6676         }
6677
6678         return amdgpu_ring_test_ring(ring);
6679 }
6680
6681 static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
6682 {
6683         struct amdgpu_device *adev = ip_block->adev;
6684         uint32_t i, j, k, reg, index = 0;
6685         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
6686
6687         if (!adev->gfx.ip_dump_core)
6688                 return;
6689
6690         for (i = 0; i < reg_count; i++)
6691                 drm_printf(p, "%-50s \t 0x%08x\n",
6692                            gc_reg_list_11_0[i].reg_name,
6693                            adev->gfx.ip_dump_core[i]);
6694
6695         /* print compute queue registers for all instances */
6696         if (!adev->gfx.ip_dump_compute_queues)
6697                 return;
6698
6699         reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
6700         drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
6701                    adev->gfx.mec.num_mec,
6702                    adev->gfx.mec.num_pipe_per_mec,
6703                    adev->gfx.mec.num_queue_per_pipe);
6704
6705         for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6706                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6707                         for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
6708                                 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
6709                                 for (reg = 0; reg < reg_count; reg++) {
6710                                         drm_printf(p, "%-50s \t 0x%08x\n",
6711                                                    gc_cp_reg_list_11[reg].reg_name,
6712                                                    adev->gfx.ip_dump_compute_queues[index + reg]);
6713                                 }
6714                                 index += reg_count;
6715                         }
6716                 }
6717         }
6718
6719         /* print gfx queue registers for all instances */
6720         if (!adev->gfx.ip_dump_gfx_queues)
6721                 return;
6722
6723         index = 0;
6724         reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
6725         drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n",
6726                    adev->gfx.me.num_me,
6727                    adev->gfx.me.num_pipe_per_me,
6728                    adev->gfx.me.num_queue_per_pipe);
6729
6730         for (i = 0; i < adev->gfx.me.num_me; i++) {
6731                 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6732                         for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
6733                                 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k);
6734                                 for (reg = 0; reg < reg_count; reg++) {
6735                                         drm_printf(p, "%-50s \t 0x%08x\n",
6736                                                    gc_gfx_queue_reg_list_11[reg].reg_name,
6737                                                    adev->gfx.ip_dump_gfx_queues[index + reg]);
6738                                 }
6739                                 index += reg_count;
6740                         }
6741                 }
6742         }
6743 }
6744
6745 static void gfx_v11_ip_dump(struct amdgpu_ip_block *ip_block)
6746 {
6747         struct amdgpu_device *adev = ip_block->adev;
6748         uint32_t i, j, k, reg, index = 0;
6749         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
6750
6751         if (!adev->gfx.ip_dump_core)
6752                 return;
6753
6754         amdgpu_gfx_off_ctrl(adev, false);
6755         for (i = 0; i < reg_count; i++)
6756                 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i]));
6757         amdgpu_gfx_off_ctrl(adev, true);
6758
6759         /* dump compute queue registers for all instances */
6760         if (!adev->gfx.ip_dump_compute_queues)
6761                 return;
6762
6763         reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
6764         amdgpu_gfx_off_ctrl(adev, false);
6765         mutex_lock(&adev->srbm_mutex);
6766         for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6767                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6768                         for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
6769                                 /* ME0 is for GFX so start from 1 for CP */
6770                                 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0);
6771                                 for (reg = 0; reg < reg_count; reg++) {
6772                                         adev->gfx.ip_dump_compute_queues[index + reg] =
6773                                                 RREG32(SOC15_REG_ENTRY_OFFSET(
6774                                                         gc_cp_reg_list_11[reg]));
6775                                 }
6776                                 index += reg_count;
6777                         }
6778                 }
6779         }
6780         soc21_grbm_select(adev, 0, 0, 0, 0);
6781         mutex_unlock(&adev->srbm_mutex);
6782         amdgpu_gfx_off_ctrl(adev, true);
6783
6784         /* dump gfx queue registers for all instances */
6785         if (!adev->gfx.ip_dump_gfx_queues)
6786                 return;
6787
6788         index = 0;
6789         reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
6790         amdgpu_gfx_off_ctrl(adev, false);
6791         mutex_lock(&adev->srbm_mutex);
6792         for (i = 0; i < adev->gfx.me.num_me; i++) {
6793                 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6794                         for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
6795                                 soc21_grbm_select(adev, i, j, k, 0);
6796
6797                                 for (reg = 0; reg < reg_count; reg++) {
6798                                         adev->gfx.ip_dump_gfx_queues[index + reg] =
6799                                                 RREG32(SOC15_REG_ENTRY_OFFSET(
6800                                                         gc_gfx_queue_reg_list_11[reg]));
6801                                 }
6802                                 index += reg_count;
6803                         }
6804                 }
6805         }
6806         soc21_grbm_select(adev, 0, 0, 0, 0);
6807         mutex_unlock(&adev->srbm_mutex);
6808         amdgpu_gfx_off_ctrl(adev, true);
6809 }
6810
6811 static void gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
6812 {
6813         /* Emit the cleaner shader */
6814         amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
6815         amdgpu_ring_write(ring, 0);  /* RESERVED field, programmed to zero */
6816 }
6817
6818 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
6819         .name = "gfx_v11_0",
6820         .early_init = gfx_v11_0_early_init,
6821         .late_init = gfx_v11_0_late_init,
6822         .sw_init = gfx_v11_0_sw_init,
6823         .sw_fini = gfx_v11_0_sw_fini,
6824         .hw_init = gfx_v11_0_hw_init,
6825         .hw_fini = gfx_v11_0_hw_fini,
6826         .suspend = gfx_v11_0_suspend,
6827         .resume = gfx_v11_0_resume,
6828         .is_idle = gfx_v11_0_is_idle,
6829         .wait_for_idle = gfx_v11_0_wait_for_idle,
6830         .soft_reset = gfx_v11_0_soft_reset,
6831         .check_soft_reset = gfx_v11_0_check_soft_reset,
6832         .post_soft_reset = gfx_v11_0_post_soft_reset,
6833         .set_clockgating_state = gfx_v11_0_set_clockgating_state,
6834         .set_powergating_state = gfx_v11_0_set_powergating_state,
6835         .get_clockgating_state = gfx_v11_0_get_clockgating_state,
6836         .dump_ip_state = gfx_v11_ip_dump,
6837         .print_ip_state = gfx_v11_ip_print,
6838 };
6839
6840 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
6841         .type = AMDGPU_RING_TYPE_GFX,
6842         .align_mask = 0xff,
6843         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6844         .support_64bit_ptrs = true,
6845         .secure_submission_supported = true,
6846         .get_rptr = gfx_v11_0_ring_get_rptr_gfx,
6847         .get_wptr = gfx_v11_0_ring_get_wptr_gfx,
6848         .set_wptr = gfx_v11_0_ring_set_wptr_gfx,
6849         .emit_frame_size = /* totally 247 maximum if 16 IBs */
6850                 5 + /* update_spm_vmid */
6851                 5 + /* COND_EXEC */
6852                 22 + /* SET_Q_PREEMPTION_MODE */
6853                 7 + /* PIPELINE_SYNC */
6854                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6855                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6856                 4 + /* VM_FLUSH */
6857                 8 + /* FENCE for VM_FLUSH */
6858                 20 + /* GDS switch */
6859                 5 + /* COND_EXEC */
6860                 7 + /* HDP_flush */
6861                 4 + /* VGT_flush */
6862                 31 + /* DE_META */
6863                 3 + /* CNTX_CTRL */
6864                 5 + /* HDP_INVL */
6865                 22 + /* SET_Q_PREEMPTION_MODE */
6866                 8 + 8 + /* FENCE x2 */
6867                 8 + /* gfx_v11_0_emit_mem_sync */
6868                 2, /* gfx_v11_0_ring_emit_cleaner_shader */
6869         .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */
6870         .emit_ib = gfx_v11_0_ring_emit_ib_gfx,
6871         .emit_fence = gfx_v11_0_ring_emit_fence,
6872         .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6873         .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6874         .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6875         .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6876         .test_ring = gfx_v11_0_ring_test_ring,
6877         .test_ib = gfx_v11_0_ring_test_ib,
6878         .insert_nop = gfx_v11_ring_insert_nop,
6879         .pad_ib = amdgpu_ring_generic_pad_ib,
6880         .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
6881         .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow,
6882         .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
6883         .preempt_ib = gfx_v11_0_ring_preempt_ib,
6884         .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
6885         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6886         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6887         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6888         .soft_recovery = gfx_v11_0_ring_soft_recovery,
6889         .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6890         .reset = gfx_v11_0_reset_kgq,
6891         .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader,
6892         .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
6893         .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
6894 };
6895
6896 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {
6897         .type = AMDGPU_RING_TYPE_COMPUTE,
6898         .align_mask = 0xff,
6899         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6900         .support_64bit_ptrs = true,
6901         .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6902         .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6903         .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6904         .emit_frame_size =
6905                 5 + /* update_spm_vmid */
6906                 20 + /* gfx_v11_0_ring_emit_gds_switch */
6907                 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6908                 5 + /* hdp invalidate */
6909                 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6910                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6911                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6912                 2 + /* gfx_v11_0_ring_emit_vm_flush */
6913                 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */
6914                 8 + /* gfx_v11_0_emit_mem_sync */
6915                 2, /* gfx_v11_0_ring_emit_cleaner_shader */
6916         .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6917         .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6918         .emit_fence = gfx_v11_0_ring_emit_fence,
6919         .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6920         .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6921         .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6922         .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6923         .test_ring = gfx_v11_0_ring_test_ring,
6924         .test_ib = gfx_v11_0_ring_test_ib,
6925         .insert_nop = gfx_v11_ring_insert_nop,
6926         .pad_ib = amdgpu_ring_generic_pad_ib,
6927         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6928         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6929         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6930         .soft_recovery = gfx_v11_0_ring_soft_recovery,
6931         .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6932         .reset = gfx_v11_0_reset_kcq,
6933         .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader,
6934         .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
6935         .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
6936 };
6937
6938 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
6939         .type = AMDGPU_RING_TYPE_KIQ,
6940         .align_mask = 0xff,
6941         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6942         .support_64bit_ptrs = true,
6943         .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6944         .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6945         .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6946         .emit_frame_size =
6947                 20 + /* gfx_v11_0_ring_emit_gds_switch */
6948                 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6949                 5 + /*hdp invalidate */
6950                 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6951                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6952                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6953                 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6954         .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6955         .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6956         .emit_fence = gfx_v11_0_ring_emit_fence_kiq,
6957         .test_ring = gfx_v11_0_ring_test_ring,
6958         .test_ib = gfx_v11_0_ring_test_ib,
6959         .insert_nop = amdgpu_ring_insert_nop,
6960         .pad_ib = amdgpu_ring_generic_pad_ib,
6961         .emit_rreg = gfx_v11_0_ring_emit_rreg,
6962         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6963         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6964         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6965 };
6966
6967 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev)
6968 {
6969         int i;
6970
6971         adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq;
6972
6973         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6974                 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx;
6975
6976         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6977                 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute;
6978 }
6979
6980 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = {
6981         .set = gfx_v11_0_set_eop_interrupt_state,
6982         .process = gfx_v11_0_eop_irq,
6983 };
6984
6985 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = {
6986         .set = gfx_v11_0_set_priv_reg_fault_state,
6987         .process = gfx_v11_0_priv_reg_irq,
6988 };
6989
6990 static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = {
6991         .set = gfx_v11_0_set_bad_op_fault_state,
6992         .process = gfx_v11_0_bad_op_irq,
6993 };
6994
6995 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = {
6996         .set = gfx_v11_0_set_priv_inst_fault_state,
6997         .process = gfx_v11_0_priv_inst_irq,
6998 };
6999
7000 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = {
7001         .process = gfx_v11_0_rlc_gc_fed_irq,
7002 };
7003
7004 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev)
7005 {
7006         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7007         adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs;
7008
7009         adev->gfx.priv_reg_irq.num_types = 1;
7010         adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs;
7011
7012         adev->gfx.bad_op_irq.num_types = 1;
7013         adev->gfx.bad_op_irq.funcs = &gfx_v11_0_bad_op_irq_funcs;
7014
7015         adev->gfx.priv_inst_irq.num_types = 1;
7016         adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs;
7017
7018         adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */
7019         adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs;
7020
7021 }
7022
7023 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev)
7024 {
7025         if (adev->flags & AMD_IS_APU)
7026                 adev->gfx.imu.mode = MISSION_MODE;
7027         else
7028                 adev->gfx.imu.mode = DEBUG_MODE;
7029
7030         adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs;
7031 }
7032
7033 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev)
7034 {
7035         adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs;
7036 }
7037
7038 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev)
7039 {
7040         unsigned total_cu = adev->gfx.config.max_cu_per_sh *
7041                             adev->gfx.config.max_sh_per_se *
7042                             adev->gfx.config.max_shader_engines;
7043
7044         adev->gds.gds_size = 0x1000;
7045         adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1;
7046         adev->gds.gws_size = 64;
7047         adev->gds.oa_size = 16;
7048 }
7049
7050 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev)
7051 {
7052         /* set gfx eng mqd */
7053         adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
7054                 sizeof(struct v11_gfx_mqd);
7055         adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
7056                 gfx_v11_0_gfx_mqd_init;
7057         /* set compute eng mqd */
7058         adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
7059                 sizeof(struct v11_compute_mqd);
7060         adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
7061                 gfx_v11_0_compute_mqd_init;
7062 }
7063
7064 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
7065                                                           u32 bitmap)
7066 {
7067         u32 data;
7068
7069         if (!bitmap)
7070                 return;
7071
7072         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
7073         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
7074
7075         WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
7076 }
7077
7078 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
7079 {
7080         u32 data, wgp_bitmask;
7081         data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
7082         data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
7083
7084         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
7085         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
7086
7087         wgp_bitmask =
7088                 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
7089
7090         return (~data) & wgp_bitmask;
7091 }
7092
7093 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
7094 {
7095         u32 wgp_idx, wgp_active_bitmap;
7096         u32 cu_bitmap_per_wgp, cu_active_bitmap;
7097
7098         wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev);
7099         cu_active_bitmap = 0;
7100
7101         for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
7102                 /* if there is one WGP enabled, it means 2 CUs will be enabled */
7103                 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
7104                 if (wgp_active_bitmap & (1 << wgp_idx))
7105                         cu_active_bitmap |= cu_bitmap_per_wgp;
7106         }
7107
7108         return cu_active_bitmap;
7109 }
7110
7111 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
7112                                  struct amdgpu_cu_info *cu_info)
7113 {
7114         int i, j, k, counter, active_cu_number = 0;
7115         u32 mask, bitmap;
7116         unsigned disable_masks[8 * 2];
7117
7118         if (!adev || !cu_info)
7119                 return -EINVAL;
7120
7121         amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
7122
7123         mutex_lock(&adev->grbm_idx_mutex);
7124         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7125                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7126                         bitmap = i * adev->gfx.config.max_sh_per_se + j;
7127                         if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1))
7128                                 continue;
7129                         mask = 1;
7130                         counter = 0;
7131                         gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0);
7132                         if (i < 8 && j < 2)
7133                                 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(
7134                                         adev, disable_masks[i * 2 + j]);
7135                         bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev);
7136
7137                         /**
7138                          * GFX11 could support more than 4 SEs, while the bitmap
7139                          * in cu_info struct is 4x4 and ioctl interface struct
7140                          * drm_amdgpu_info_device should keep stable.
7141                          * So we use last two columns of bitmap to store cu mask for
7142                          * SEs 4 to 7, the layout of the bitmap is as below:
7143                          *    SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
7144                          *    SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
7145                          *    SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
7146                          *    SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
7147                          *    SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
7148                          *    SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
7149                          *    SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
7150                          *    SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
7151                          */
7152                         cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap;
7153
7154                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
7155                                 if (bitmap & mask)
7156                                         counter++;
7157
7158                                 mask <<= 1;
7159                         }
7160                         active_cu_number += counter;
7161                 }
7162         }
7163         gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7164         mutex_unlock(&adev->grbm_idx_mutex);
7165
7166         cu_info->number = active_cu_number;
7167         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7168
7169         return 0;
7170 }
7171
7172 const struct amdgpu_ip_block_version gfx_v11_0_ip_block =
7173 {
7174         .type = AMD_IP_BLOCK_TYPE_GFX,
7175         .major = 11,
7176         .minor = 0,
7177         .rev = 0,
7178         .funcs = &gfx_v11_0_ip_funcs,
7179 };