drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "soc15.h"
  33 #include "soc15d.h"
  34 #include "amdgpu_atomfirmware.h"
  35 #include "amdgpu_pm.h"
  36
  37 #include "gc/gc_9_0_offset.h"
  38 #include "gc/gc_9_0_sh_mask.h"
  39
  40 #include "vega10_enum.h"
  41
  42 #include "soc15_common.h"
  43 #include "clearstate_gfx9.h"
  44 #include "v9_structs.h"
  45
  46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
  47
  48 #include "amdgpu_ras.h"
  49
  50 #include "amdgpu_ring_mux.h"
  51 #include "gfx_v9_4.h"
  52 #include "gfx_v9_0.h"
  53 #include "gfx_v9_0_cleaner_shader.h"
  54 #include "gfx_v9_4_2.h"
  55
  56 #include "asic_reg/pwr/pwr_10_0_offset.h"
  57 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
  58 #include "asic_reg/gc/gc_9_0_default.h"
  59
  60 #define GFX9_NUM_GFX_RINGS     1
  61 #define GFX9_NUM_SW_GFX_RINGS  2
  62 #define GFX9_MEC_HPD_SIZE 4096
  63 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
  64 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
  65
  66 #define mmGCEA_PROBE_MAP                        0x070c
  67 #define mmGCEA_PROBE_MAP_BASE_IDX               0
  68
  69 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  70 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  71 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  72 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  73 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  74 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  75
  76 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
  77 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
  78 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
  79 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
  80 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
  81 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
  82
  83 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
  84 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
  85 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
  86 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
  87 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
  88 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
  89
  90 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
  91 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
  92 MODULE_FIRMWARE("amdgpu/raven_me.bin");
  93 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
  94 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
  95 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
  96
  97 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
  98 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
  99 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
 100 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
 101 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
 102 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
 103 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
 104
 105 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
 106 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
 107 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
 108 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
 109 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
 110 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
 111 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
 112
 113 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
 114 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
 115
 116 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
 117 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
 118 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
 119 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
 120 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
 121
 122 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
 123 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
 124 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
 125 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
 126 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
 127 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
 128
 129 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
 130 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
 131 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
 132 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");
 133 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
 134
 135 #define mmTCP_CHAN_STEER_0_ARCT                                                         0x0b03
 136 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX                                                        0
 137 #define mmTCP_CHAN_STEER_1_ARCT                                                         0x0b04
 138 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX                                                        0
 139 #define mmTCP_CHAN_STEER_2_ARCT                                                         0x0b09
 140 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX                                                        0
 141 #define mmTCP_CHAN_STEER_3_ARCT                                                         0x0b0a
 142 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX                                                        0
 143 #define mmTCP_CHAN_STEER_4_ARCT                                                         0x0b0b
 144 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX                                                        0
 145 #define mmTCP_CHAN_STEER_5_ARCT                                                         0x0b0c
 146 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX                                                        0
 147
 148 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir                0x0025
 149 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX       1
 150 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir                0x0026
 151 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX       1
 152
 153 static const struct amdgpu_hwip_reg_entry gc_reg_list_9[] = {
 154         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS),
 155         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2),
 156         SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT1),
 157         SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT2),
 158         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STALLED_STAT1),
 159         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STALLED_STAT1),
 160         SOC15_REG_ENTRY_STR(GC, 0, mmCP_BUSY_STAT),
 161         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT),
 162         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT),
 163         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STATUS),
 164         SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_ERROR),
 165         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_BASE),
 166         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_RPTR),
 167         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_WPTR),
 168         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_BASE),
 169         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_RPTR),
 170         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_WPTR),
 171         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_BASE),
 172         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_RPTR),
 173         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_WPTR),
 174         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_BASE),
 175         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
 176         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
 177         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_CMD_BUFSZ),
 178         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_CMD_BUFSZ),
 179         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_CMD_BUFSZ),
 180         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_CMD_BUFSZ),
 181         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_LO),
 182         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_HI),
 183         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BUFSZ),
 184         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_LO),
 185         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_HI),
 186         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BUFSZ),
 187         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_LO),
 188         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_HI),
 189         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BUFSZ),
 190         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_LO),
 191         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_HI),
 192         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BUFSZ),
 193         SOC15_REG_ENTRY_STR(GC, 0, mmCPF_UTCL1_STATUS),
 194         SOC15_REG_ENTRY_STR(GC, 0, mmCPC_UTCL1_STATUS),
 195         SOC15_REG_ENTRY_STR(GC, 0, mmCPG_UTCL1_STATUS),
 196         SOC15_REG_ENTRY_STR(GC, 0, mmGDS_PROTECTION_FAULT),
 197         SOC15_REG_ENTRY_STR(GC, 0, mmGDS_VM_PROTECTION_FAULT),
 198         SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS),
 199         SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_CNTL),
 200         SOC15_REG_ENTRY_STR(GC, 0, mmPA_CL_CNTL_STATUS),
 201         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_UTCL1_STATUS),
 202         SOC15_REG_ENTRY_STR(GC, 0, mmRMI_UTCL1_STATUS),
 203         SOC15_REG_ENTRY_STR(GC, 0, mmSQC_DCACHE_UTCL1_STATUS),
 204         SOC15_REG_ENTRY_STR(GC, 0, mmSQC_ICACHE_UTCL1_STATUS),
 205         SOC15_REG_ENTRY_STR(GC, 0, mmSQ_UTCL1_STATUS),
 206         SOC15_REG_ENTRY_STR(GC, 0, mmTCP_UTCL1_STATUS),
 207         SOC15_REG_ENTRY_STR(GC, 0, mmWD_UTCL1_STATUS),
 208         SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL),
 209         SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS),
 210         SOC15_REG_ENTRY_STR(GC, 0, mmCP_DEBUG),
 211         SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_CNTL),
 212         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_INSTR_PNTR),
 213         SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC1_INSTR_PNTR),
 214         SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC2_INSTR_PNTR),
 215         SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_INSTR_PNTR),
 216         SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_INSTR_PNTR),
 217         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STATUS),
 218         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_STAT),
 219         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_COMMAND),
 220         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_MESSAGE),
 221         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_1),
 222         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_2),
 223         SOC15_REG_ENTRY_STR(GC, 0, mmSMU_RLC_RESPONSE),
 224         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SAFE_MODE),
 225         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_SAFE_MODE),
 226         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_INT_STAT),
 227         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_GENERAL_6),
 228         /* cp header registers */
 229         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
 230         SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
 231         SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME2_HEADER_DUMP),
 232         SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
 233         SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
 234         /* SE status registers */
 235         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE0),
 236         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE1),
 237         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE2),
 238         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE3)
 239 };
 240
 241 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9[] = {
 242         /* compute queue registers */
 243         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_VMID),
 244         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ACTIVE),
 245         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PERSISTENT_STATE),
 246         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PIPE_PRIORITY),
 247         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUEUE_PRIORITY),
 248         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUANTUM),
 249         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE),
 250         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE_HI),
 251         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_RPTR),
 252         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
 253         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
 254         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL),
 255         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_CONTROL),
 256         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR),
 257         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR_HI),
 258         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_RPTR),
 259         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_CONTROL),
 260         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_DEQUEUE_REQUEST),
 261         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR),
 262         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI),
 263         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_CONTROL),
 264         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_RPTR),
 265         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR),
 266         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_EVENTS),
 267         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_LO),
 268         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_HI),
 269         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_CONTROL),
 270         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_OFFSET),
 271         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_SIZE),
 272         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_WG_STATE_OFFSET),
 273         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_SIZE),
 274         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GDS_RESOURCE_STATE),
 275         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ERROR),
 276         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR_MEM),
 277         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_LO),
 278         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_HI),
 279         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GFX_STATUS),
 280 };
 281
 282 enum ta_ras_gfx_subblock {
 283         /*CPC*/
 284         TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
 285         TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
 286         TA_RAS_BLOCK__GFX_CPC_UCODE,
 287         TA_RAS_BLOCK__GFX_DC_STATE_ME1,
 288         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
 289         TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
 290         TA_RAS_BLOCK__GFX_DC_STATE_ME2,
 291         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
 292         TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 293         TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 294         /* CPF*/
 295         TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 296         TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 297         TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
 298         TA_RAS_BLOCK__GFX_CPF_TAG,
 299         TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
 300         /* CPG*/
 301         TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 302         TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 303         TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
 304         TA_RAS_BLOCK__GFX_CPG_TAG,
 305         TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
 306         /* GDS*/
 307         TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 308         TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 309         TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
 310         TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
 311         TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
 312         TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 313         TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 314         /* SPI*/
 315         TA_RAS_BLOCK__GFX_SPI_SR_MEM,
 316         /* SQ*/
 317         TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 318         TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 319         TA_RAS_BLOCK__GFX_SQ_LDS_D,
 320         TA_RAS_BLOCK__GFX_SQ_LDS_I,
 321         TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
 322         TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
 323         /* SQC (3 ranges)*/
 324         TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 325         /* SQC range 0*/
 326         TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 327         TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
 328                 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
 329         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
 330         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
 331         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
 332         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
 333         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
 334         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 335         TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
 336                 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 337         /* SQC range 1*/
 338         TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 339         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
 340                 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 341         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
 342         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
 343         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
 344         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
 345         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
 346         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
 347         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
 348         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 349         TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
 350                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 351         /* SQC range 2*/
 352         TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 353         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
 354                 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 355         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
 356         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
 357         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
 358         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
 359         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
 360         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
 361         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
 362         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 363         TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
 364                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 365         TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
 366         /* TA*/
 367         TA_RAS_BLOCK__GFX_TA_INDEX_START,
 368         TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
 369         TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
 370         TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
 371         TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
 372         TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 373         TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 374         /* TCA*/
 375         TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 376         TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 377         TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 378         TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 379         /* TCC (5 sub-ranges)*/
 380         TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 381         /* TCC range 0*/
 382         TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 383         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
 384         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
 385         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
 386         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
 387         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
 388         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
 389         TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
 390         TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 391         TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 392         /* TCC range 1*/
 393         TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 394         TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 395         TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 396         TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
 397                 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 398         /* TCC range 2*/
 399         TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 400         TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 401         TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
 402         TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
 403         TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
 404         TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
 405         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
 406         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
 407         TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 408         TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
 409                 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 410         /* TCC range 3*/
 411         TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 412         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 413         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 414         TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
 415                 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 416         /* TCC range 4*/
 417         TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 418         TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
 419                 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 420         TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 421         TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
 422                 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 423         TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
 424         /* TCI*/
 425         TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
 426         /* TCP*/
 427         TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 428         TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 429         TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
 430         TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
 431         TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
 432         TA_RAS_BLOCK__GFX_TCP_DB_RAM,
 433         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
 434         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 435         TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 436         /* TD*/
 437         TA_RAS_BLOCK__GFX_TD_INDEX_START,
 438         TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
 439         TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
 440         TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 441         TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 442         /* EA (3 sub-ranges)*/
 443         TA_RAS_BLOCK__GFX_EA_INDEX_START,
 444         /* EA range 0*/
 445         TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
 446         TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
 447         TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
 448         TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
 449         TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
 450         TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
 451         TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
 452         TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
 453         TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 454         TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 455         /* EA range 1*/
 456         TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 457         TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 458         TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
 459         TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
 460         TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
 461         TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
 462         TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
 463         TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 464         TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 465         /* EA range 2*/
 466         TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 467         TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 468         TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
 469         TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
 470         TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 471         TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 472         TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
 473         /* UTC VM L2 bank*/
 474         TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
 475         /* UTC VM walker*/
 476         TA_RAS_BLOCK__UTC_VML2_WALKER,
 477         /* UTC ATC L2 2MB cache*/
 478         TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
 479         /* UTC ATC L2 4KB cache*/
 480         TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
 481         TA_RAS_BLOCK__GFX_MAX
 482 };
 483
 484 struct ras_gfx_subblock {
 485         unsigned char *name;
 486         int ta_subblock;
 487         int hw_supported_error_type;
 488         int sw_supported_error_type;
 489 };
 490
 491 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \
 492         [AMDGPU_RAS_BLOCK__##subblock] = {                                     \
 493                 #subblock,                                                     \
 494                 TA_RAS_BLOCK__##subblock,                                      \
 495                 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \
 496                 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \
 497         }
 498
 499 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
 500         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
 501         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
 502         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 503         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 504         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 505         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 506         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 507         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 508         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 509         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 510         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
 511         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
 512         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
 513         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
 514         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 515         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
 516         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
 517                              0),
 518         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
 519                              0),
 520         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 521         AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 522         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 523         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
 524         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
 525         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 526         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
 527         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 528                              0, 0),
 529         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 530                              0),
 531         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 532                              0, 0),
 533         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
 534                              0),
 535         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 536                              0, 0),
 537         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 538                              0),
 539         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 540                              1),
 541         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 542                              0, 0, 0),
 543         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 544                              0),
 545         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 546                              0),
 547         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 548                              0),
 549         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 550                              0),
 551         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 552                              0),
 553         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 554                              0, 0),
 555         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 556                              0),
 557         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 558                              0),
 559         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 560                              0, 0, 0),
 561         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 562                              0),
 563         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 564                              0),
 565         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 566                              0),
 567         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 568                              0),
 569         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 570                              0),
 571         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 572                              0, 0),
 573         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 574                              0),
 575         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
 576         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 577         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 578         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 579         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 580         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
 581         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 582         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
 583         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
 584                              1),
 585         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
 586                              1),
 587         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
 588                              1),
 589         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
 590                              0),
 591         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
 592                              0),
 593         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 594         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 595         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
 596         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
 597         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
 598         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
 599         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 600         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
 601         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
 602         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 603         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
 604         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
 605                              0),
 606         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 607         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
 608                              0),
 609         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
 610                              0, 0),
 611         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
 612                              0),
 613         AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 614         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
 615         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
 616         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 617         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 618         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 619         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
 620         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
 621         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
 622         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
 623         AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 624         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
 625         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 626         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 627         AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 628         AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 629         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 630         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 631         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 632         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 633         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 634         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 635         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 636         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 637         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 638         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 639         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 640         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 641         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 642         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 643         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
 644         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
 645         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
 646         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
 647 };
 648
 649 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
 650 {
 651         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 652         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
 653         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 654         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 655         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 656         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 657         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 658         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 659         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 660         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
 661         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
 662         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 663         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 664         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 665         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 666         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 667         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
 668         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 669         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 670         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 671 };
 672
 673 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 674 {
 675         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 676         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 677         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 678         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 679         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 680         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 681         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
 682         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 683         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
 684         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 685         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 686         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 687         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 688         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 689         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 690         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
 691         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
 692         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 693 };
 694
 695 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
 696 {
 697         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
 698         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 699         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 700         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
 701         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
 702         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
 703         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
 704         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
 705         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
 706         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
 707         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
 708 };
 709
 710 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 711 {
 712         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 713         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 714         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 715         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 716         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 717         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 718         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 719         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 720         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 721         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 722         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 723         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 724         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 725         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 726         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 727         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 728         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 729         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 730         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 731         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
 732         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
 733         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 734         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 735         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 736 };
 737
 738 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 739 {
 740         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 741         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
 742         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
 743         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
 744         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
 745         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 746         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 747 };
 748
 749 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
 750 {
 751         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
 752         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 753         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 754         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
 755         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
 756         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
 757         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
 758         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
 759         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 760         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 761         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
 762         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
 763         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
 764         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 765         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 766         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 767         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
 768         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 769         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 770 };
 771
 772 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
 773 {
 774         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 775         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 776         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 777         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
 778         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
 779         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 780         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 781         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 782         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 783         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 784         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 785         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
 786 };
 787
 788 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 789 {
 790         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
 791         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
 792         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 793 };
 794
 795 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
 796 {
 797         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 798         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 799         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 800         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 801         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 802         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 803         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 804         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 805         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
 806         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 807         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 808         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 809         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 810         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 811         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 812         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 813 };
 814
 815 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 816 {
 817         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
 818         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 819         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 820         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
 821         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
 822         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 823         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
 824         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 825         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
 826         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 827         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 828         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 829         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 830 };
 831
 832 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
 833 {
 834         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 835         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
 836         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
 837         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
 838         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
 839         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
 840         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
 841         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
 842         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
 843         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
 844         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
 845 };
 846
 847 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
 848         {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
 849         {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
 850 };
 851
 852 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
 853 {
 854         mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 855         mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 856         mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 857         mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 858         mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 859         mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 860         mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 861         mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 862 };
 863
 864 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 865 {
 866         mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 867         mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 868         mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 869         mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 870         mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 871         mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 872         mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 873         mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 874 };
 875
 876 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 877 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 878 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 879 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
 880
 881 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 882 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 883 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 884 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 885 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 886                                 struct amdgpu_cu_info *cu_info);
 887 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 888 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);
 889 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
 890 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
 891                                           void *ras_error_status);
 892 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
 893                                      void *inject_if, uint32_t instance_mask);
 894 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
 895 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
 896                                               unsigned int vmid);
 897 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
 898 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
 899
 900 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 901                                 uint64_t queue_mask)
 902 {
 903         struct amdgpu_device *adev = kiq_ring->adev;
 904         u64 shader_mc_addr;
 905
 906         /* Cleaner shader MC address */
 907         shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
 908
 909         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 910         amdgpu_ring_write(kiq_ring,
 911                 PACKET3_SET_RESOURCES_VMID_MASK(0) |
 912                 /* vmid_mask:0* queue_type:0 (KIQ) */
 913                 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
 914         amdgpu_ring_write(kiq_ring,
 915                         lower_32_bits(queue_mask));     /* queue mask lo */
 916         amdgpu_ring_write(kiq_ring,
 917                         upper_32_bits(queue_mask));     /* queue mask hi */
 918         amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
 919         amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
 920         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 921         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 922 }
 923
 924 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 925                                  struct amdgpu_ring *ring)
 926 {
 927         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 928         uint64_t wptr_addr = ring->wptr_gpu_addr;
 929         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 930
 931         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 932         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 933         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 934                          PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 935                          PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 936                          PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 937                          PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 938                          PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
 939                          /*queue_type: normal compute queue */
 940                          PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
 941                          /* alloc format: all_on_one_pipe */
 942                          PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
 943                          PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 944                          /* num_queues: must be 1 */
 945                          PACKET3_MAP_QUEUES_NUM_QUEUES(1));
 946         amdgpu_ring_write(kiq_ring,
 947                         PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 948         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 949         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 950         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 951         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 952 }
 953
 954 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 955                                    struct amdgpu_ring *ring,
 956                                    enum amdgpu_unmap_queues_action action,
 957                                    u64 gpu_addr, u64 seq)
 958 {
 959         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 960
 961         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 962         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 963                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 964                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 965                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 966                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 967         amdgpu_ring_write(kiq_ring,
 968                         PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 969
 970         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 971                 amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));
 972                 amdgpu_ring_write(kiq_ring, 0);
 973                 amdgpu_ring_write(kiq_ring, 0);
 974
 975         } else {
 976                 amdgpu_ring_write(kiq_ring, 0);
 977                 amdgpu_ring_write(kiq_ring, 0);
 978                 amdgpu_ring_write(kiq_ring, 0);
 979         }
 980 }
 981
 982 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 983                                    struct amdgpu_ring *ring,
 984                                    u64 addr,
 985                                    u64 seq)
 986 {
 987         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 988
 989         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 990         amdgpu_ring_write(kiq_ring,
 991                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 992                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 993                           PACKET3_QUERY_STATUS_COMMAND(2));
 994         /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 995         amdgpu_ring_write(kiq_ring,
 996                         PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 997                         PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 998         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 999         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
1000         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
1001         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
1002 }
1003
1004 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
1005                                 uint16_t pasid, uint32_t flush_type,
1006                                 bool all_hub)
1007 {
1008         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
1009         amdgpu_ring_write(kiq_ring,
1010                         PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
1011                         PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
1012                         PACKET3_INVALIDATE_TLBS_PASID(pasid) |
1013                         PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
1014 }
1015
1016
1017 static void gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type,
1018                                         uint32_t me_id, uint32_t pipe_id, uint32_t queue_id,
1019                                         uint32_t xcc_id, uint32_t vmid)
1020 {
1021         struct amdgpu_device *adev = kiq_ring->adev;
1022         unsigned i;
1023
1024         /* enter save mode */
1025         amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
1026         mutex_lock(&adev->srbm_mutex);
1027         soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, 0);
1028
1029         if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
1030                 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0x2);
1031                 WREG32_SOC15(GC, 0, mmSPI_COMPUTE_QUEUE_RESET, 0x1);
1032                 /* wait till dequeue take effects */
1033                 for (i = 0; i < adev->usec_timeout; i++) {
1034                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
1035                                 break;
1036                         udelay(1);
1037                 }
1038                 if (i >= adev->usec_timeout)
1039                         dev_err(adev->dev, "fail to wait on hqd deactive\n");
1040         } else {
1041                 dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type);
1042         }
1043
1044         soc15_grbm_select(adev, 0, 0, 0, 0, 0);
1045         mutex_unlock(&adev->srbm_mutex);
1046         /* exit safe mode */
1047         amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
1048 }
1049
1050 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
1051         .kiq_set_resources = gfx_v9_0_kiq_set_resources,
1052         .kiq_map_queues = gfx_v9_0_kiq_map_queues,
1053         .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
1054         .kiq_query_status = gfx_v9_0_kiq_query_status,
1055         .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
1056         .kiq_reset_hw_queue = gfx_v9_0_kiq_reset_hw_queue,
1057         .set_resources_size = 8,
1058         .map_queues_size = 7,
1059         .unmap_queues_size = 6,
1060         .query_status_size = 7,
1061         .invalidate_tlbs_size = 2,
1062 };
1063
1064 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
1065 {
1066         adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs;
1067 }
1068
1069 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
1070 {
1071         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1072         case IP_VERSION(9, 0, 1):
1073                 soc15_program_register_sequence(adev,
1074                                                 golden_settings_gc_9_0,
1075                                                 ARRAY_SIZE(golden_settings_gc_9_0));
1076                 soc15_program_register_sequence(adev,
1077                                                 golden_settings_gc_9_0_vg10,
1078                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
1079                 break;
1080         case IP_VERSION(9, 2, 1):
1081                 soc15_program_register_sequence(adev,
1082                                                 golden_settings_gc_9_2_1,
1083                                                 ARRAY_SIZE(golden_settings_gc_9_2_1));
1084                 soc15_program_register_sequence(adev,
1085                                                 golden_settings_gc_9_2_1_vg12,
1086                                                 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
1087                 break;
1088         case IP_VERSION(9, 4, 0):
1089                 soc15_program_register_sequence(adev,
1090                                                 golden_settings_gc_9_0,
1091                                                 ARRAY_SIZE(golden_settings_gc_9_0));
1092                 soc15_program_register_sequence(adev,
1093                                                 golden_settings_gc_9_0_vg20,
1094                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
1095                 break;
1096         case IP_VERSION(9, 4, 1):
1097                 soc15_program_register_sequence(adev,
1098                                                 golden_settings_gc_9_4_1_arct,
1099                                                 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
1100                 break;
1101         case IP_VERSION(9, 2, 2):
1102         case IP_VERSION(9, 1, 0):
1103                 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
1104                                                 ARRAY_SIZE(golden_settings_gc_9_1));
1105                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1106                         soc15_program_register_sequence(adev,
1107                                                         golden_settings_gc_9_1_rv2,
1108                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv2));
1109                 else
1110                         soc15_program_register_sequence(adev,
1111                                                         golden_settings_gc_9_1_rv1,
1112                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv1));
1113                 break;
1114          case IP_VERSION(9, 3, 0):
1115                 soc15_program_register_sequence(adev,
1116                                                 golden_settings_gc_9_1_rn,
1117                                                 ARRAY_SIZE(golden_settings_gc_9_1_rn));
1118                 return; /* for renoir, don't need common goldensetting */
1119         case IP_VERSION(9, 4, 2):
1120                 gfx_v9_4_2_init_golden_registers(adev,
1121                                                  adev->smuio.funcs->get_die_id(adev));
1122                 break;
1123         default:
1124                 break;
1125         }
1126
1127         if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1128             (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)))
1129                 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
1130                                                 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
1131 }
1132
1133 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1134                                        bool wc, uint32_t reg, uint32_t val)
1135 {
1136         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1137         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1138                                 WRITE_DATA_DST_SEL(0) |
1139                                 (wc ? WR_CONFIRM : 0));
1140         amdgpu_ring_write(ring, reg);
1141         amdgpu_ring_write(ring, 0);
1142         amdgpu_ring_write(ring, val);
1143 }
1144
1145 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1146                                   int mem_space, int opt, uint32_t addr0,
1147                                   uint32_t addr1, uint32_t ref, uint32_t mask,
1148                                   uint32_t inv)
1149 {
1150         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1151         amdgpu_ring_write(ring,
1152                                  /* memory (1) or register (0) */
1153                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1154                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
1155                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
1156                                  WAIT_REG_MEM_ENGINE(eng_sel)));
1157
1158         if (mem_space)
1159                 BUG_ON(addr0 & 0x3); /* Dword align */
1160         amdgpu_ring_write(ring, addr0);
1161         amdgpu_ring_write(ring, addr1);
1162         amdgpu_ring_write(ring, ref);
1163         amdgpu_ring_write(ring, mask);
1164         amdgpu_ring_write(ring, inv); /* poll interval */
1165 }
1166
1167 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1168 {
1169         struct amdgpu_device *adev = ring->adev;
1170         uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1171         uint32_t tmp = 0;
1172         unsigned i;
1173         int r;
1174
1175         WREG32(scratch, 0xCAFEDEAD);
1176         r = amdgpu_ring_alloc(ring, 3);
1177         if (r)
1178                 return r;
1179
1180         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1181         amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START);
1182         amdgpu_ring_write(ring, 0xDEADBEEF);
1183         amdgpu_ring_commit(ring);
1184
1185         for (i = 0; i < adev->usec_timeout; i++) {
1186                 tmp = RREG32(scratch);
1187                 if (tmp == 0xDEADBEEF)
1188                         break;
1189                 udelay(1);
1190         }
1191
1192         if (i >= adev->usec_timeout)
1193                 r = -ETIMEDOUT;
1194         return r;
1195 }
1196
1197 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1198 {
1199         struct amdgpu_device *adev = ring->adev;
1200         struct amdgpu_ib ib;
1201         struct dma_fence *f = NULL;
1202
1203         unsigned index;
1204         uint64_t gpu_addr;
1205         uint32_t tmp;
1206         long r;
1207
1208         r = amdgpu_device_wb_get(adev, &index);
1209         if (r)
1210                 return r;
1211
1212         gpu_addr = adev->wb.gpu_addr + (index * 4);
1213         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1214         memset(&ib, 0, sizeof(ib));
1215
1216         r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
1217         if (r)
1218                 goto err1;
1219
1220         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1221         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1222         ib.ptr[2] = lower_32_bits(gpu_addr);
1223         ib.ptr[3] = upper_32_bits(gpu_addr);
1224         ib.ptr[4] = 0xDEADBEEF;
1225         ib.length_dw = 5;
1226
1227         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1228         if (r)
1229                 goto err2;
1230
1231         r = dma_fence_wait_timeout(f, false, timeout);
1232         if (r == 0) {
1233                 r = -ETIMEDOUT;
1234                 goto err2;
1235         } else if (r < 0) {
1236                 goto err2;
1237         }
1238
1239         tmp = adev->wb.wb[index];
1240         if (tmp == 0xDEADBEEF)
1241                 r = 0;
1242         else
1243                 r = -EINVAL;
1244
1245 err2:
1246         amdgpu_ib_free(adev, &ib, NULL);
1247         dma_fence_put(f);
1248 err1:
1249         amdgpu_device_wb_free(adev, index);
1250         return r;
1251 }
1252
1253
1254 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1255 {
1256         amdgpu_ucode_release(&adev->gfx.pfp_fw);
1257         amdgpu_ucode_release(&adev->gfx.me_fw);
1258         amdgpu_ucode_release(&adev->gfx.ce_fw);
1259         amdgpu_ucode_release(&adev->gfx.rlc_fw);
1260         amdgpu_ucode_release(&adev->gfx.mec_fw);
1261         amdgpu_ucode_release(&adev->gfx.mec2_fw);
1262
1263         kfree(adev->gfx.rlc.register_list_format);
1264 }
1265
1266 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1267 {
1268         adev->gfx.me_fw_write_wait = false;
1269         adev->gfx.mec_fw_write_wait = false;
1270
1271         if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1272             ((adev->gfx.mec_fw_version < 0x000001a5) ||
1273              (adev->gfx.mec_feature_version < 46) ||
1274              (adev->gfx.pfp_fw_version < 0x000000b7) ||
1275              (adev->gfx.pfp_feature_version < 46)))
1276                 DRM_WARN_ONCE("CP firmware version too old, please update!");
1277
1278         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1279         case IP_VERSION(9, 0, 1):
1280                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1281                     (adev->gfx.me_feature_version >= 42) &&
1282                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1283                     (adev->gfx.pfp_feature_version >= 42))
1284                         adev->gfx.me_fw_write_wait = true;
1285
1286                 if ((adev->gfx.mec_fw_version >=  0x00000193) &&
1287                     (adev->gfx.mec_feature_version >= 42))
1288                         adev->gfx.mec_fw_write_wait = true;
1289                 break;
1290         case IP_VERSION(9, 2, 1):
1291                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1292                     (adev->gfx.me_feature_version >= 44) &&
1293                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1294                     (adev->gfx.pfp_feature_version >= 44))
1295                         adev->gfx.me_fw_write_wait = true;
1296
1297                 if ((adev->gfx.mec_fw_version >=  0x00000196) &&
1298                     (adev->gfx.mec_feature_version >= 44))
1299                         adev->gfx.mec_fw_write_wait = true;
1300                 break;
1301         case IP_VERSION(9, 4, 0):
1302                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1303                     (adev->gfx.me_feature_version >= 44) &&
1304                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1305                     (adev->gfx.pfp_feature_version >= 44))
1306                         adev->gfx.me_fw_write_wait = true;
1307
1308                 if ((adev->gfx.mec_fw_version >=  0x00000197) &&
1309                     (adev->gfx.mec_feature_version >= 44))
1310                         adev->gfx.mec_fw_write_wait = true;
1311                 break;
1312         case IP_VERSION(9, 1, 0):
1313         case IP_VERSION(9, 2, 2):
1314                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1315                     (adev->gfx.me_feature_version >= 42) &&
1316                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1317                     (adev->gfx.pfp_feature_version >= 42))
1318                         adev->gfx.me_fw_write_wait = true;
1319
1320                 if ((adev->gfx.mec_fw_version >=  0x00000192) &&
1321                     (adev->gfx.mec_feature_version >= 42))
1322                         adev->gfx.mec_fw_write_wait = true;
1323                 break;
1324         default:
1325                 adev->gfx.me_fw_write_wait = true;
1326                 adev->gfx.mec_fw_write_wait = true;
1327                 break;
1328         }
1329 }
1330
1331 struct amdgpu_gfxoff_quirk {
1332         u16 chip_vendor;
1333         u16 chip_device;
1334         u16 subsys_vendor;
1335         u16 subsys_device;
1336         u8 revision;
1337 };
1338
1339 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1340         /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1341         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1342         /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1343         { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1344         /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1345         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1346         /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1347         { 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
1348         /* https://bbs.openkylin.top/t/topic/171497 */
1349         { 0x1002, 0x15d8, 0x19e5, 0x3e14, 0xc2 },
1350         /* HP 705G4 DM with R5 2400G */
1351         { 0x1002, 0x15dd, 0x103c, 0x8464, 0xd6 },
1352         { 0, 0, 0, 0, 0 },
1353 };
1354
1355 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1356 {
1357         const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1358
1359         while (p && p->chip_device != 0) {
1360                 if (pdev->vendor == p->chip_vendor &&
1361                     pdev->device == p->chip_device &&
1362                     pdev->subsystem_vendor == p->subsys_vendor &&
1363                     pdev->subsystem_device == p->subsys_device &&
1364                     pdev->revision == p->revision) {
1365                         return true;
1366                 }
1367                 ++p;
1368         }
1369         return false;
1370 }
1371
1372 static bool is_raven_kicker(struct amdgpu_device *adev)
1373 {
1374         if (adev->pm.fw_version >= 0x41e2b)
1375                 return true;
1376         else
1377                 return false;
1378 }
1379
1380 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1381 {
1382         if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) &&
1383             (adev->gfx.me_fw_version >= 0x000000a5) &&
1384             (adev->gfx.me_feature_version >= 52))
1385                 return true;
1386         else
1387                 return false;
1388 }
1389
1390 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1391 {
1392         if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1393                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1394
1395         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1396         case IP_VERSION(9, 0, 1):
1397         case IP_VERSION(9, 2, 1):
1398         case IP_VERSION(9, 4, 0):
1399                 break;
1400         case IP_VERSION(9, 2, 2):
1401         case IP_VERSION(9, 1, 0):
1402                 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1403                       (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1404                     ((!is_raven_kicker(adev) &&
1405                       adev->gfx.rlc_fw_version < 531) ||
1406                      (adev->gfx.rlc_feature_version < 1) ||
1407                      !adev->gfx.rlc.is_rlc_v2_1))
1408                         adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1409
1410                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1411                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1412                                 AMD_PG_SUPPORT_CP |
1413                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1414                 break;
1415         case IP_VERSION(9, 3, 0):
1416                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1417                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1418                                 AMD_PG_SUPPORT_CP |
1419                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1420                 break;
1421         default:
1422                 break;
1423         }
1424 }
1425
1426 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1427                                           char *chip_name)
1428 {
1429         int err;
1430
1431         err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
1432                                    "amdgpu/%s_pfp.bin", chip_name);
1433         if (err)
1434                 goto out;
1435         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
1436
1437         err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
1438                                    "amdgpu/%s_me.bin", chip_name);
1439         if (err)
1440                 goto out;
1441         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
1442
1443         err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,
1444                                    "amdgpu/%s_ce.bin", chip_name);
1445         if (err)
1446                 goto out;
1447         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE);
1448
1449 out:
1450         if (err) {
1451                 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1452                 amdgpu_ucode_release(&adev->gfx.me_fw);
1453                 amdgpu_ucode_release(&adev->gfx.ce_fw);
1454         }
1455         return err;
1456 }
1457
1458 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1459                                        char *chip_name)
1460 {
1461         int err;
1462         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1463         uint16_t version_major;
1464         uint16_t version_minor;
1465         uint32_t smu_version;
1466
1467         /*
1468          * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1469          * instead of picasso_rlc.bin.
1470          * Judgment method:
1471          * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1472          *          or revision >= 0xD8 && revision <= 0xDF
1473          * otherwise is PCO FP5
1474          */
1475         if (!strcmp(chip_name, "picasso") &&
1476                 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1477                 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1478                 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1479                                            "amdgpu/%s_rlc_am4.bin", chip_name);
1480         else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1481                 (smu_version >= 0x41e2b))
1482                 /**
1483                 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1484                 */
1485                 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1486                                            "amdgpu/%s_kicker_rlc.bin", chip_name);
1487         else
1488                 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1489                                            "amdgpu/%s_rlc.bin", chip_name);
1490         if (err)
1491                 goto out;
1492
1493         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1494         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1495         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1496         err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
1497 out:
1498         if (err)
1499                 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1500
1501         return err;
1502 }
1503
1504 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1505 {
1506         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
1507             amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
1508             amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0))
1509                 return false;
1510
1511         return true;
1512 }
1513
1514 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1515                                               char *chip_name)
1516 {
1517         int err;
1518
1519         if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1520                 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1521                                            "amdgpu/%s_sjt_mec.bin", chip_name);
1522         else
1523                 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1524                                            "amdgpu/%s_mec.bin", chip_name);
1525         if (err)
1526                 goto out;
1527
1528         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
1529         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
1530
1531         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1532                 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1533                         err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1534                                                    "amdgpu/%s_sjt_mec2.bin", chip_name);
1535                 else
1536                         err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1537                                                    "amdgpu/%s_mec2.bin", chip_name);
1538                 if (!err) {
1539                         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2);
1540                         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT);
1541                 } else {
1542                         err = 0;
1543                         amdgpu_ucode_release(&adev->gfx.mec2_fw);
1544                 }
1545         } else {
1546                 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1547                 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1548         }
1549
1550         gfx_v9_0_check_if_need_gfxoff(adev);
1551         gfx_v9_0_check_fw_write_wait(adev);
1552
1553 out:
1554         if (err)
1555                 amdgpu_ucode_release(&adev->gfx.mec_fw);
1556         return err;
1557 }
1558
1559 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1560 {
1561         char ucode_prefix[30];
1562         int r;
1563
1564         DRM_DEBUG("\n");
1565         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
1566
1567         /* No CPG in Arcturus */
1568         if (adev->gfx.num_gfx_rings) {
1569                 r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix);
1570                 if (r)
1571                         return r;
1572         }
1573
1574         r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix);
1575         if (r)
1576                 return r;
1577
1578         r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix);
1579         if (r)
1580                 return r;
1581
1582         return r;
1583 }
1584
1585 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1586 {
1587         u32 count = 0;
1588         const struct cs_section_def *sect = NULL;
1589         const struct cs_extent_def *ext = NULL;
1590
1591         /* begin clear state */
1592         count += 2;
1593         /* context control state */
1594         count += 3;
1595
1596         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1597                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1598                         if (sect->id == SECT_CONTEXT)
1599                                 count += 2 + ext->reg_count;
1600                         else
1601                                 return 0;
1602                 }
1603         }
1604
1605         /* end clear state */
1606         count += 2;
1607         /* clear state */
1608         count += 2;
1609
1610         return count;
1611 }
1612
1613 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1614                                     volatile u32 *buffer)
1615 {
1616         u32 count = 0, i;
1617         const struct cs_section_def *sect = NULL;
1618         const struct cs_extent_def *ext = NULL;
1619
1620         if (adev->gfx.rlc.cs_data == NULL)
1621                 return;
1622         if (buffer == NULL)
1623                 return;
1624
1625         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1626         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1627
1628         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1629         buffer[count++] = cpu_to_le32(0x80000000);
1630         buffer[count++] = cpu_to_le32(0x80000000);
1631
1632         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1633                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1634                         if (sect->id == SECT_CONTEXT) {
1635                                 buffer[count++] =
1636                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1637                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1638                                                 PACKET3_SET_CONTEXT_REG_START);
1639                                 for (i = 0; i < ext->reg_count; i++)
1640                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1641                         } else {
1642                                 return;
1643                         }
1644                 }
1645         }
1646
1647         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1648         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1649
1650         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1651         buffer[count++] = cpu_to_le32(0);
1652 }
1653
1654 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1655 {
1656         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1657         uint32_t pg_always_on_cu_num = 2;
1658         uint32_t always_on_cu_num;
1659         uint32_t i, j, k;
1660         uint32_t mask, cu_bitmap, counter;
1661
1662         if (adev->flags & AMD_IS_APU)
1663                 always_on_cu_num = 4;
1664         else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1))
1665                 always_on_cu_num = 8;
1666         else
1667                 always_on_cu_num = 12;
1668
1669         mutex_lock(&adev->grbm_idx_mutex);
1670         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1671                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1672                         mask = 1;
1673                         cu_bitmap = 0;
1674                         counter = 0;
1675                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
1676
1677                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1678                                 if (cu_info->bitmap[0][i][j] & mask) {
1679                                         if (counter == pg_always_on_cu_num)
1680                                                 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1681                                         if (counter < always_on_cu_num)
1682                                                 cu_bitmap |= mask;
1683                                         else
1684                                                 break;
1685                                         counter++;
1686                                 }
1687                                 mask <<= 1;
1688                         }
1689
1690                         WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1691                         cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1692                 }
1693         }
1694         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1695         mutex_unlock(&adev->grbm_idx_mutex);
1696 }
1697
1698 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1699 {
1700         uint32_t data;
1701
1702         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1703         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1704         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1705         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1706         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1707
1708         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1709         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1710
1711         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1712         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1713
1714         mutex_lock(&adev->grbm_idx_mutex);
1715         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1716         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1717         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1718
1719         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1720         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1721         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1722         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1723         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1724
1725         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1726         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1727         data &= 0x0000FFFF;
1728         data |= 0x00C00000;
1729         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1730
1731         /*
1732          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1733          * programmed in gfx_v9_0_init_always_on_cu_mask()
1734          */
1735
1736         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1737          * but used for RLC_LB_CNTL configuration */
1738         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1739         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1740         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1741         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1742         mutex_unlock(&adev->grbm_idx_mutex);
1743
1744         gfx_v9_0_init_always_on_cu_mask(adev);
1745 }
1746
1747 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1748 {
1749         uint32_t data;
1750
1751         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1752         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1753         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1754         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1755         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1756
1757         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1758         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1759
1760         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1761         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1762
1763         mutex_lock(&adev->grbm_idx_mutex);
1764         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1765         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1766         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1767
1768         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1769         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1770         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1771         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1772         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1773
1774         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1775         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1776         data &= 0x0000FFFF;
1777         data |= 0x00C00000;
1778         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1779
1780         /*
1781          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1782          * programmed in gfx_v9_0_init_always_on_cu_mask()
1783          */
1784
1785         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1786          * but used for RLC_LB_CNTL configuration */
1787         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1788         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1789         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1790         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1791         mutex_unlock(&adev->grbm_idx_mutex);
1792
1793         gfx_v9_0_init_always_on_cu_mask(adev);
1794 }
1795
1796 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1797 {
1798         WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1799 }
1800
1801 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1802 {
1803         if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1804                 return 5;
1805         else
1806                 return 4;
1807 }
1808
1809 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1810 {
1811         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1812
1813         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
1814         reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1815         reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);
1816         reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);
1817         reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);
1818         reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);
1819         reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);
1820         reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);
1821         adev->gfx.rlc.rlcg_reg_access_supported = true;
1822 }
1823
1824 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1825 {
1826         const struct cs_section_def *cs_data;
1827         int r;
1828
1829         adev->gfx.rlc.cs_data = gfx9_cs_data;
1830
1831         cs_data = adev->gfx.rlc.cs_data;
1832
1833         if (cs_data) {
1834                 /* init clear state block */
1835                 r = amdgpu_gfx_rlc_init_csb(adev);
1836                 if (r)
1837                         return r;
1838         }
1839
1840         if (adev->flags & AMD_IS_APU) {
1841                 /* TODO: double check the cp_table_size for RV */
1842                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1843                 r = amdgpu_gfx_rlc_init_cpt(adev);
1844                 if (r)
1845                         return r;
1846         }
1847
1848         return 0;
1849 }
1850
1851 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1852 {
1853         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1854         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1855 }
1856
1857 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1858 {
1859         int r;
1860         u32 *hpd;
1861         const __le32 *fw_data;
1862         unsigned fw_size;
1863         u32 *fw;
1864         size_t mec_hpd_size;
1865
1866         const struct gfx_firmware_header_v1_0 *mec_hdr;
1867
1868         bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1869
1870         /* take ownership of the relevant compute queues */
1871         amdgpu_gfx_compute_queue_acquire(adev);
1872         mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1873         if (mec_hpd_size) {
1874                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1875                                               AMDGPU_GEM_DOMAIN_VRAM |
1876                                               AMDGPU_GEM_DOMAIN_GTT,
1877                                               &adev->gfx.mec.hpd_eop_obj,
1878                                               &adev->gfx.mec.hpd_eop_gpu_addr,
1879                                               (void **)&hpd);
1880                 if (r) {
1881                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1882                         gfx_v9_0_mec_fini(adev);
1883                         return r;
1884                 }
1885
1886                 memset(hpd, 0, mec_hpd_size);
1887
1888                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1889                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1890         }
1891
1892         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1893
1894         fw_data = (const __le32 *)
1895                 (adev->gfx.mec_fw->data +
1896                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1897         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1898
1899         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1900                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1901                                       &adev->gfx.mec.mec_fw_obj,
1902                                       &adev->gfx.mec.mec_fw_gpu_addr,
1903                                       (void **)&fw);
1904         if (r) {
1905                 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1906                 gfx_v9_0_mec_fini(adev);
1907                 return r;
1908         }
1909
1910         memcpy(fw, fw_data, fw_size);
1911
1912         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1913         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1914
1915         return 0;
1916 }
1917
1918 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1919 {
1920         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1921                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1922                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1923                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
1924                 (SQ_IND_INDEX__FORCE_READ_MASK));
1925         return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1926 }
1927
1928 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1929                            uint32_t wave, uint32_t thread,
1930                            uint32_t regno, uint32_t num, uint32_t *out)
1931 {
1932         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1933                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1934                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1935                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1936                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1937                 (SQ_IND_INDEX__FORCE_READ_MASK) |
1938                 (SQ_IND_INDEX__AUTO_INCR_MASK));
1939         while (num--)
1940                 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1941 }
1942
1943 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1944 {
1945         /* type 1 wave data */
1946         dst[(*no_fields)++] = 1;
1947         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1948         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1949         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1950         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1951         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1952         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1953         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1954         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1955         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1956         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1957         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1958         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1959         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1960         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1961         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
1962 }
1963
1964 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1965                                      uint32_t wave, uint32_t start,
1966                                      uint32_t size, uint32_t *dst)
1967 {
1968         wave_read_regs(
1969                 adev, simd, wave, 0,
1970                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
1971 }
1972
1973 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1974                                      uint32_t wave, uint32_t thread,
1975                                      uint32_t start, uint32_t size,
1976                                      uint32_t *dst)
1977 {
1978         wave_read_regs(
1979                 adev, simd, wave, thread,
1980                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1981 }
1982
1983 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
1984                                   u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1985 {
1986         soc15_grbm_select(adev, me, pipe, q, vm, 0);
1987 }
1988
1989 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
1990         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
1991         .select_se_sh = &gfx_v9_0_select_se_sh,
1992         .read_wave_data = &gfx_v9_0_read_wave_data,
1993         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
1994         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
1995         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
1996 };
1997
1998 const struct amdgpu_ras_block_hw_ops  gfx_v9_0_ras_ops = {
1999                 .ras_error_inject = &gfx_v9_0_ras_error_inject,
2000                 .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2001                 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2002 };
2003
2004 static struct amdgpu_gfx_ras gfx_v9_0_ras = {
2005         .ras_block = {
2006                 .hw_ops = &gfx_v9_0_ras_ops,
2007         },
2008 };
2009
2010 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2011 {
2012         u32 gb_addr_config;
2013         int err;
2014
2015         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2016         case IP_VERSION(9, 0, 1):
2017                 adev->gfx.config.max_hw_contexts = 8;
2018                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2019                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2020                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2021                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2022                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2023                 break;
2024         case IP_VERSION(9, 2, 1):
2025                 adev->gfx.config.max_hw_contexts = 8;
2026                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2027                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2028                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2029                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2030                 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2031                 DRM_INFO("fix gfx.config for vega12\n");
2032                 break;
2033         case IP_VERSION(9, 4, 0):
2034                 adev->gfx.ras = &gfx_v9_0_ras;
2035                 adev->gfx.config.max_hw_contexts = 8;
2036                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2037                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2038                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2039                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2040                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2041                 gb_addr_config &= ~0xf3e777ff;
2042                 gb_addr_config |= 0x22014042;
2043                 /* check vbios table if gpu info is not available */
2044                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2045                 if (err)
2046                         return err;
2047                 break;
2048         case IP_VERSION(9, 2, 2):
2049         case IP_VERSION(9, 1, 0):
2050                 adev->gfx.config.max_hw_contexts = 8;
2051                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2052                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2053                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2054                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2055                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2056                         gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2057                 else
2058                         gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2059                 break;
2060         case IP_VERSION(9, 4, 1):
2061                 adev->gfx.ras = &gfx_v9_4_ras;
2062                 adev->gfx.config.max_hw_contexts = 8;
2063                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2064                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2065                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2066                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2067                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2068                 gb_addr_config &= ~0xf3e777ff;
2069                 gb_addr_config |= 0x22014042;
2070                 break;
2071         case IP_VERSION(9, 3, 0):
2072                 adev->gfx.config.max_hw_contexts = 8;
2073                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2074                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2075                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2076                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2077                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2078                 gb_addr_config &= ~0xf3e777ff;
2079                 gb_addr_config |= 0x22010042;
2080                 break;
2081         case IP_VERSION(9, 4, 2):
2082                 adev->gfx.ras = &gfx_v9_4_2_ras;
2083                 adev->gfx.config.max_hw_contexts = 8;
2084                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2085                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2086                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2087                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2088                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2089                 gb_addr_config &= ~0xf3e777ff;
2090                 gb_addr_config |= 0x22014042;
2091                 /* check vbios table if gpu info is not available */
2092                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2093                 if (err)
2094                         return err;
2095                 break;
2096         default:
2097                 BUG();
2098                 break;
2099         }
2100
2101         adev->gfx.config.gb_addr_config = gb_addr_config;
2102
2103         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2104                         REG_GET_FIELD(
2105                                         adev->gfx.config.gb_addr_config,
2106                                         GB_ADDR_CONFIG,
2107                                         NUM_PIPES);
2108
2109         adev->gfx.config.max_tile_pipes =
2110                 adev->gfx.config.gb_addr_config_fields.num_pipes;
2111
2112         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2113                         REG_GET_FIELD(
2114                                         adev->gfx.config.gb_addr_config,
2115                                         GB_ADDR_CONFIG,
2116                                         NUM_BANKS);
2117         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2118                         REG_GET_FIELD(
2119                                         adev->gfx.config.gb_addr_config,
2120                                         GB_ADDR_CONFIG,
2121                                         MAX_COMPRESSED_FRAGS);
2122         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2123                         REG_GET_FIELD(
2124                                         adev->gfx.config.gb_addr_config,
2125                                         GB_ADDR_CONFIG,
2126                                         NUM_RB_PER_SE);
2127         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2128                         REG_GET_FIELD(
2129                                         adev->gfx.config.gb_addr_config,
2130                                         GB_ADDR_CONFIG,
2131                                         NUM_SHADER_ENGINES);
2132         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2133                         REG_GET_FIELD(
2134                                         adev->gfx.config.gb_addr_config,
2135                                         GB_ADDR_CONFIG,
2136                                         PIPE_INTERLEAVE_SIZE));
2137
2138         return 0;
2139 }
2140
2141 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2142                                       int mec, int pipe, int queue)
2143 {
2144         unsigned irq_type;
2145         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2146         unsigned int hw_prio;
2147
2148         ring = &adev->gfx.compute_ring[ring_id];
2149
2150         /* mec0 is me1 */
2151         ring->me = mec + 1;
2152         ring->pipe = pipe;
2153         ring->queue = queue;
2154
2155         ring->ring_obj = NULL;
2156         ring->use_doorbell = true;
2157         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2158         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2159                                 + (ring_id * GFX9_MEC_HPD_SIZE);
2160         ring->vm_hub = AMDGPU_GFXHUB(0);
2161         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2162
2163         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2164                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2165                 + ring->pipe;
2166         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2167                         AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
2168         /* type-2 packets are deprecated on MEC, use type-3 instead */
2169         return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2170                                 hw_prio, NULL);
2171 }
2172
2173 static void gfx_v9_0_alloc_ip_dump(struct amdgpu_device *adev)
2174 {
2175         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
2176         uint32_t *ptr;
2177         uint32_t inst;
2178
2179         ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
2180         if (!ptr) {
2181                 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
2182                 adev->gfx.ip_dump_core = NULL;
2183         } else {
2184                 adev->gfx.ip_dump_core = ptr;
2185         }
2186
2187         /* Allocate memory for compute queue registers for all the instances */
2188         reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
2189         inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
2190                 adev->gfx.mec.num_queue_per_pipe;
2191
2192         ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
2193         if (!ptr) {
2194                 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
2195                 adev->gfx.ip_dump_compute_queues = NULL;
2196         } else {
2197                 adev->gfx.ip_dump_compute_queues = ptr;
2198         }
2199 }
2200
2201 static int gfx_v9_0_sw_init(void *handle)
2202 {
2203         int i, j, k, r, ring_id;
2204         int xcc_id = 0;
2205         struct amdgpu_ring *ring;
2206         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2207         unsigned int hw_prio;
2208
2209         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2210         case IP_VERSION(9, 0, 1):
2211         case IP_VERSION(9, 2, 1):
2212         case IP_VERSION(9, 4, 0):
2213         case IP_VERSION(9, 2, 2):
2214         case IP_VERSION(9, 1, 0):
2215         case IP_VERSION(9, 4, 1):
2216         case IP_VERSION(9, 3, 0):
2217         case IP_VERSION(9, 4, 2):
2218                 adev->gfx.mec.num_mec = 2;
2219                 break;
2220         default:
2221                 adev->gfx.mec.num_mec = 1;
2222                 break;
2223         }
2224
2225         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2226         default:
2227                 adev->gfx.enable_cleaner_shader = false;
2228                 break;
2229         }
2230
2231         adev->gfx.mec.num_pipe_per_mec = 4;
2232         adev->gfx.mec.num_queue_per_pipe = 8;
2233
2234         /* EOP Event */
2235         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2236         if (r)
2237                 return r;
2238
2239         /* Bad opcode Event */
2240         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
2241                               GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR,
2242                               &adev->gfx.bad_op_irq);
2243         if (r)
2244                 return r;
2245
2246         /* Privileged reg */
2247         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2248                               &adev->gfx.priv_reg_irq);
2249         if (r)
2250                 return r;
2251
2252         /* Privileged inst */
2253         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2254                               &adev->gfx.priv_inst_irq);
2255         if (r)
2256                 return r;
2257
2258         /* ECC error */
2259         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2260                               &adev->gfx.cp_ecc_error_irq);
2261         if (r)
2262                 return r;
2263
2264         /* FUE error */
2265         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2266                               &adev->gfx.cp_ecc_error_irq);
2267         if (r)
2268                 return r;
2269
2270         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2271
2272         if (adev->gfx.rlc.funcs) {
2273                 if (adev->gfx.rlc.funcs->init) {
2274                         r = adev->gfx.rlc.funcs->init(adev);
2275                         if (r) {
2276                                 dev_err(adev->dev, "Failed to init rlc BOs!\n");
2277                                 return r;
2278                         }
2279                 }
2280         }
2281
2282         r = gfx_v9_0_mec_init(adev);
2283         if (r) {
2284                 DRM_ERROR("Failed to init MEC BOs!\n");
2285                 return r;
2286         }
2287
2288         /* set up the gfx ring */
2289         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2290                 ring = &adev->gfx.gfx_ring[i];
2291                 ring->ring_obj = NULL;
2292                 if (!i)
2293                         sprintf(ring->name, "gfx");
2294                 else
2295                         sprintf(ring->name, "gfx_%d", i);
2296                 ring->use_doorbell = true;
2297                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2298
2299                 /* disable scheduler on the real ring */
2300                 ring->no_scheduler = adev->gfx.mcbp;
2301                 ring->vm_hub = AMDGPU_GFXHUB(0);
2302                 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2303                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2304                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
2305                 if (r)
2306                         return r;
2307         }
2308
2309         /* set up the software rings */
2310         if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2311                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2312                         ring = &adev->gfx.sw_gfx_ring[i];
2313                         ring->ring_obj = NULL;
2314                         sprintf(ring->name, amdgpu_sw_ring_name(i));
2315                         ring->use_doorbell = true;
2316                         ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2317                         ring->is_sw_ring = true;
2318                         hw_prio = amdgpu_sw_ring_priority(i);
2319                         ring->vm_hub = AMDGPU_GFXHUB(0);
2320                         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2321                                              AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,
2322                                              NULL);
2323                         if (r)
2324                                 return r;
2325                         ring->wptr = 0;
2326                 }
2327
2328                 /* init the muxer and add software rings */
2329                 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0],
2330                                          GFX9_NUM_SW_GFX_RINGS);
2331                 if (r) {
2332                         DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r);
2333                         return r;
2334                 }
2335                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2336                         r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer,
2337                                                         &adev->gfx.sw_gfx_ring[i]);
2338                         if (r) {
2339                                 DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r);
2340                                 return r;
2341                         }
2342                 }
2343         }
2344
2345         /* set up the compute queues - allocate horizontally across pipes */
2346         ring_id = 0;
2347         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2348                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2349                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2350                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
2351                                                                      k, j))
2352                                         continue;
2353
2354                                 r = gfx_v9_0_compute_ring_init(adev,
2355                                                                ring_id,
2356                                                                i, k, j);
2357                                 if (r)
2358                                         return r;
2359
2360                                 ring_id++;
2361                         }
2362                 }
2363         }
2364
2365         r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0);
2366         if (r) {
2367                 DRM_ERROR("Failed to init KIQ BOs!\n");
2368                 return r;
2369         }
2370
2371         r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
2372         if (r)
2373                 return r;
2374
2375         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2376         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation), 0);
2377         if (r)
2378                 return r;
2379
2380         adev->gfx.ce_ram_size = 0x8000;
2381
2382         r = gfx_v9_0_gpu_early_init(adev);
2383         if (r)
2384                 return r;
2385
2386         if (amdgpu_gfx_ras_sw_init(adev)) {
2387                 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
2388                 return -EINVAL;
2389         }
2390
2391         gfx_v9_0_alloc_ip_dump(adev);
2392
2393         r = amdgpu_gfx_sysfs_isolation_shader_init(adev);
2394         if (r)
2395                 return r;
2396
2397         return 0;
2398 }
2399
2400
2401 static int gfx_v9_0_sw_fini(void *handle)
2402 {
2403         int i;
2404         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2405
2406         if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2407                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
2408                         amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]);
2409                 amdgpu_ring_mux_fini(&adev->gfx.muxer);
2410         }
2411
2412         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2413                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2414         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2415                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2416
2417         amdgpu_gfx_mqd_sw_fini(adev, 0);
2418         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
2419         amdgpu_gfx_kiq_fini(adev, 0);
2420
2421         gfx_v9_0_mec_fini(adev);
2422         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2423                                 &adev->gfx.rlc.clear_state_gpu_addr,
2424                                 (void **)&adev->gfx.rlc.cs_ptr);
2425         if (adev->flags & AMD_IS_APU) {
2426                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2427                                 &adev->gfx.rlc.cp_table_gpu_addr,
2428                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2429         }
2430         gfx_v9_0_free_microcode(adev);
2431
2432         amdgpu_gfx_sysfs_isolation_shader_fini(adev);
2433
2434         kfree(adev->gfx.ip_dump_core);
2435         kfree(adev->gfx.ip_dump_compute_queues);
2436
2437         return 0;
2438 }
2439
2440
2441 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2442 {
2443         /* TODO */
2444 }
2445
2446 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2447                            u32 instance, int xcc_id)
2448 {
2449         u32 data;
2450
2451         if (instance == 0xffffffff)
2452                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2453         else
2454                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2455
2456         if (se_num == 0xffffffff)
2457                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2458         else
2459                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2460
2461         if (sh_num == 0xffffffff)
2462                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2463         else
2464                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2465
2466         WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2467 }
2468
2469 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2470 {
2471         u32 data, mask;
2472
2473         data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2474         data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2475
2476         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2477         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2478
2479         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2480                                          adev->gfx.config.max_sh_per_se);
2481
2482         return (~data) & mask;
2483 }
2484
2485 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2486 {
2487         int i, j;
2488         u32 data;
2489         u32 active_rbs = 0;
2490         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2491                                         adev->gfx.config.max_sh_per_se;
2492
2493         mutex_lock(&adev->grbm_idx_mutex);
2494         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2495                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2496                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2497                         data = gfx_v9_0_get_rb_active_bitmap(adev);
2498                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2499                                                rb_bitmap_width_per_sh);
2500                 }
2501         }
2502         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2503         mutex_unlock(&adev->grbm_idx_mutex);
2504
2505         adev->gfx.config.backend_enable_mask = active_rbs;
2506         adev->gfx.config.num_rbs = hweight32(active_rbs);
2507 }
2508
2509 static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
2510                                 uint32_t first_vmid,
2511                                 uint32_t last_vmid)
2512 {
2513         uint32_t data;
2514         uint32_t trap_config_vmid_mask = 0;
2515         int i;
2516
2517         /* Calculate trap config vmid mask */
2518         for (i = first_vmid; i < last_vmid; i++)
2519                 trap_config_vmid_mask |= (1 << i);
2520
2521         data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
2522                         VMID_SEL, trap_config_vmid_mask);
2523         data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
2524                         TRAP_EN, 1);
2525         WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
2526         WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
2527
2528         WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0);
2529         WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0);
2530 }
2531
2532 #define DEFAULT_SH_MEM_BASES    (0x6000)
2533 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2534 {
2535         int i;
2536         uint32_t sh_mem_config;
2537         uint32_t sh_mem_bases;
2538
2539         /*
2540          * Configure apertures:
2541          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
2542          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
2543          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
2544          */
2545         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2546
2547         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2548                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2549                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2550
2551         mutex_lock(&adev->srbm_mutex);
2552         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2553                 soc15_grbm_select(adev, 0, 0, 0, i, 0);
2554                 /* CP and shaders */
2555                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2556                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2557         }
2558         soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2559         mutex_unlock(&adev->srbm_mutex);
2560
2561         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2562            access. These should be enabled by FW for target VMIDs. */
2563         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2564                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2565                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2566                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2567                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2568         }
2569 }
2570
2571 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2572 {
2573         int vmid;
2574
2575         /*
2576          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2577          * access. Compute VMIDs should be enabled by FW for target VMIDs,
2578          * the driver can enable them for graphics. VMID0 should maintain
2579          * access so that HWS firmware can save/restore entries.
2580          */
2581         for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2582                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2583                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2584                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2585                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2586         }
2587 }
2588
2589 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2590 {
2591         uint32_t tmp;
2592
2593         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2594         case IP_VERSION(9, 4, 1):
2595                 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2596                 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT,
2597                                 !READ_ONCE(adev->barrier_has_auto_waitcnt));
2598                 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2599                 break;
2600         default:
2601                 break;
2602         }
2603 }
2604
2605 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2606 {
2607         u32 tmp;
2608         int i;
2609
2610         WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2611
2612         gfx_v9_0_tiling_mode_table_init(adev);
2613
2614         if (adev->gfx.num_gfx_rings)
2615                 gfx_v9_0_setup_rb(adev);
2616         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2617         adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2618
2619         /* XXX SH_MEM regs */
2620         /* where to put LDS, scratch, GPUVM in FSA64 space */
2621         mutex_lock(&adev->srbm_mutex);
2622         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
2623                 soc15_grbm_select(adev, 0, 0, 0, i, 0);
2624                 /* CP and shaders */
2625                 if (i == 0) {
2626                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2627                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2628                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2629                                             !!adev->gmc.noretry);
2630                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2631                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2632                 } else {
2633                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2634                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2635                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2636                                             !!adev->gmc.noretry);
2637                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2638                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2639                                 (adev->gmc.private_aperture_start >> 48));
2640                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2641                                 (adev->gmc.shared_aperture_start >> 48));
2642                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2643                 }
2644         }
2645         soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2646
2647         mutex_unlock(&adev->srbm_mutex);
2648
2649         gfx_v9_0_init_compute_vmid(adev);
2650         gfx_v9_0_init_gds_vmid(adev);
2651         gfx_v9_0_init_sq_config(adev);
2652 }
2653
2654 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2655 {
2656         u32 i, j, k;
2657         u32 mask;
2658
2659         mutex_lock(&adev->grbm_idx_mutex);
2660         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2661                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2662                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2663                         for (k = 0; k < adev->usec_timeout; k++) {
2664                                 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2665                                         break;
2666                                 udelay(1);
2667                         }
2668                         if (k == adev->usec_timeout) {
2669                                 amdgpu_gfx_select_se_sh(adev, 0xffffffff,
2670                                                       0xffffffff, 0xffffffff, 0);
2671                                 mutex_unlock(&adev->grbm_idx_mutex);
2672                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2673                                          i, j);
2674                                 return;
2675                         }
2676                 }
2677         }
2678         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2679         mutex_unlock(&adev->grbm_idx_mutex);
2680
2681         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2682                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2683                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2684                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2685         for (k = 0; k < adev->usec_timeout; k++) {
2686                 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2687                         break;
2688                 udelay(1);
2689         }
2690 }
2691
2692 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2693                                                bool enable)
2694 {
2695         u32 tmp;
2696
2697         /* These interrupts should be enabled to drive DS clock */
2698
2699         tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2700
2701         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2702         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2703         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2704         if (adev->gfx.num_gfx_rings)
2705                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2706
2707         WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2708 }
2709
2710 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2711 {
2712         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2713         /* csib */
2714         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2715                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
2716         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2717                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2718         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2719                         adev->gfx.rlc.clear_state_size);
2720 }
2721
2722 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2723                                 int indirect_offset,
2724                                 int list_size,
2725                                 int *unique_indirect_regs,
2726                                 int unique_indirect_reg_count,
2727                                 int *indirect_start_offsets,
2728                                 int *indirect_start_offsets_count,
2729                                 int max_start_offsets_count)
2730 {
2731         int idx;
2732
2733         for (; indirect_offset < list_size; indirect_offset++) {
2734                 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2735                 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2736                 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2737
2738                 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2739                         indirect_offset += 2;
2740
2741                         /* look for the matching indice */
2742                         for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2743                                 if (unique_indirect_regs[idx] ==
2744                                         register_list_format[indirect_offset] ||
2745                                         !unique_indirect_regs[idx])
2746                                         break;
2747                         }
2748
2749                         BUG_ON(idx >= unique_indirect_reg_count);
2750
2751                         if (!unique_indirect_regs[idx])
2752                                 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2753
2754                         indirect_offset++;
2755                 }
2756         }
2757 }
2758
2759 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2760 {
2761         int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2762         int unique_indirect_reg_count = 0;
2763
2764         int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2765         int indirect_start_offsets_count = 0;
2766
2767         int list_size = 0;
2768         int i = 0, j = 0;
2769         u32 tmp = 0;
2770
2771         u32 *register_list_format =
2772                 kmemdup(adev->gfx.rlc.register_list_format,
2773                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2774         if (!register_list_format)
2775                 return -ENOMEM;
2776
2777         /* setup unique_indirect_regs array and indirect_start_offsets array */
2778         unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2779         gfx_v9_1_parse_ind_reg_list(register_list_format,
2780                                     adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2781                                     adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2782                                     unique_indirect_regs,
2783                                     unique_indirect_reg_count,
2784                                     indirect_start_offsets,
2785                                     &indirect_start_offsets_count,
2786                                     ARRAY_SIZE(indirect_start_offsets));
2787
2788         /* enable auto inc in case it is disabled */
2789         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2790         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2791         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2792
2793         /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2794         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2795                 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2796         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2797                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2798                         adev->gfx.rlc.register_restore[i]);
2799
2800         /* load indirect register */
2801         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2802                 adev->gfx.rlc.reg_list_format_start);
2803
2804         /* direct register portion */
2805         for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2806                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2807                         register_list_format[i]);
2808
2809         /* indirect register portion */
2810         while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2811                 if (register_list_format[i] == 0xFFFFFFFF) {
2812                         WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2813                         continue;
2814                 }
2815
2816                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2817                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2818
2819                 for (j = 0; j < unique_indirect_reg_count; j++) {
2820                         if (register_list_format[i] == unique_indirect_regs[j]) {
2821                                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2822                                 break;
2823                         }
2824                 }
2825
2826                 BUG_ON(j >= unique_indirect_reg_count);
2827
2828                 i++;
2829         }
2830
2831         /* set save/restore list size */
2832         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2833         list_size = list_size >> 1;
2834         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2835                 adev->gfx.rlc.reg_restore_list_size);
2836         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2837
2838         /* write the starting offsets to RLC scratch ram */
2839         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2840                 adev->gfx.rlc.starting_offsets_start);
2841         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2842                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2843                        indirect_start_offsets[i]);
2844
2845         /* load unique indirect regs*/
2846         for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2847                 if (unique_indirect_regs[i] != 0) {
2848                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2849                                + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2850                                unique_indirect_regs[i] & 0x3FFFF);
2851
2852                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2853                                + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2854                                unique_indirect_regs[i] >> 20);
2855                 }
2856         }
2857
2858         kfree(register_list_format);
2859         return 0;
2860 }
2861
2862 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2863 {
2864         WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2865 }
2866
2867 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2868                                              bool enable)
2869 {
2870         uint32_t data = 0;
2871         uint32_t default_data = 0;
2872
2873         default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2874         if (enable) {
2875                 /* enable GFXIP control over CGPG */
2876                 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2877                 if(default_data != data)
2878                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2879
2880                 /* update status */
2881                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2882                 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2883                 if(default_data != data)
2884                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2885         } else {
2886                 /* restore GFXIP control over GCPG */
2887                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2888                 if(default_data != data)
2889                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2890         }
2891 }
2892
2893 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2894 {
2895         uint32_t data = 0;
2896
2897         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2898                               AMD_PG_SUPPORT_GFX_SMG |
2899                               AMD_PG_SUPPORT_GFX_DMG)) {
2900                 /* init IDLE_POLL_COUNT = 60 */
2901                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2902                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2903                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2904                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2905
2906                 /* init RLC PG Delay */
2907                 data = 0;
2908                 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2909                 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2910                 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2911                 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2912                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2913
2914                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2915                 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2916                 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2917                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2918
2919                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2920                 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2921                 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2922                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2923
2924                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2925                 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2926
2927                 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2928                 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2929                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2930                 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0))
2931                         pwr_10_0_gfxip_control_over_cgpg(adev, true);
2932         }
2933 }
2934
2935 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2936                                                 bool enable)
2937 {
2938         uint32_t data = 0;
2939         uint32_t default_data = 0;
2940
2941         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2942         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2943                              SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2944                              enable ? 1 : 0);
2945         if (default_data != data)
2946                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2947 }
2948
2949 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2950                                                 bool enable)
2951 {
2952         uint32_t data = 0;
2953         uint32_t default_data = 0;
2954
2955         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2956         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2957                              SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2958                              enable ? 1 : 0);
2959         if(default_data != data)
2960                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2961 }
2962
2963 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2964                                         bool enable)
2965 {
2966         uint32_t data = 0;
2967         uint32_t default_data = 0;
2968
2969         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2970         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2971                              CP_PG_DISABLE,
2972                              enable ? 0 : 1);
2973         if(default_data != data)
2974                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2975 }
2976
2977 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2978                                                 bool enable)
2979 {
2980         uint32_t data, default_data;
2981
2982         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2983         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2984                              GFX_POWER_GATING_ENABLE,
2985                              enable ? 1 : 0);
2986         if(default_data != data)
2987                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2988 }
2989
2990 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2991                                                 bool enable)
2992 {
2993         uint32_t data, default_data;
2994
2995         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2996         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2997                              GFX_PIPELINE_PG_ENABLE,
2998                              enable ? 1 : 0);
2999         if(default_data != data)
3000                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3001
3002         if (!enable)
3003                 /* read any GFX register to wake up GFX */
3004                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
3005 }
3006
3007 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3008                                                        bool enable)
3009 {
3010         uint32_t data, default_data;
3011
3012         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3013         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3014                              STATIC_PER_CU_PG_ENABLE,
3015                              enable ? 1 : 0);
3016         if(default_data != data)
3017                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3018 }
3019
3020 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3021                                                 bool enable)
3022 {
3023         uint32_t data, default_data;
3024
3025         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3026         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3027                              DYN_PER_CU_PG_ENABLE,
3028                              enable ? 1 : 0);
3029         if(default_data != data)
3030                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3031 }
3032
3033 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3034 {
3035         gfx_v9_0_init_csb(adev);
3036
3037         /*
3038          * Rlc save restore list is workable since v2_1.
3039          * And it's needed by gfxoff feature.
3040          */
3041         if (adev->gfx.rlc.is_rlc_v2_1) {
3042                 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
3043                             IP_VERSION(9, 2, 1) ||
3044                     (adev->apu_flags & AMD_APU_IS_RAVEN2))
3045                         gfx_v9_1_init_rlc_save_restore_list(adev);
3046                 gfx_v9_0_enable_save_restore_machine(adev);
3047         }
3048
3049         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3050                               AMD_PG_SUPPORT_GFX_SMG |
3051                               AMD_PG_SUPPORT_GFX_DMG |
3052                               AMD_PG_SUPPORT_CP |
3053                               AMD_PG_SUPPORT_GDS |
3054                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
3055                 WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,
3056                              adev->gfx.rlc.cp_table_gpu_addr >> 8);
3057                 gfx_v9_0_init_gfx_power_gating(adev);
3058         }
3059 }
3060
3061 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3062 {
3063         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3064         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3065         gfx_v9_0_wait_for_rlc_serdes(adev);
3066 }
3067
3068 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3069 {
3070         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3071         udelay(50);
3072         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3073         udelay(50);
3074 }
3075
3076 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3077 {
3078 #ifdef AMDGPU_RLC_DEBUG_RETRY
3079         u32 rlc_ucode_ver;
3080 #endif
3081
3082         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3083         udelay(50);
3084
3085         /* carrizo do enable cp interrupt after cp inited */
3086         if (!(adev->flags & AMD_IS_APU)) {
3087                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3088                 udelay(50);
3089         }
3090
3091 #ifdef AMDGPU_RLC_DEBUG_RETRY
3092         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3093         rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3094         if(rlc_ucode_ver == 0x108) {
3095                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3096                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3097                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3098                  * default is 0x9C4 to create a 100us interval */
3099                 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3100                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3101                  * to disable the page fault retry interrupts, default is
3102                  * 0x100 (256) */
3103                 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3104         }
3105 #endif
3106 }
3107
3108 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3109 {
3110         const struct rlc_firmware_header_v2_0 *hdr;
3111         const __le32 *fw_data;
3112         unsigned i, fw_size;
3113
3114         if (!adev->gfx.rlc_fw)
3115                 return -EINVAL;
3116
3117         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3118         amdgpu_ucode_print_rlc_hdr(&hdr->header);
3119
3120         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3121                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3122         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3123
3124         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3125                         RLCG_UCODE_LOADING_START_ADDRESS);
3126         for (i = 0; i < fw_size; i++)
3127                 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3128         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3129
3130         return 0;
3131 }
3132
3133 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3134 {
3135         int r;
3136
3137         if (amdgpu_sriov_vf(adev)) {
3138                 gfx_v9_0_init_csb(adev);
3139                 return 0;
3140         }
3141
3142         adev->gfx.rlc.funcs->stop(adev);
3143
3144         /* disable CG */
3145         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3146
3147         gfx_v9_0_init_pg(adev);
3148
3149         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3150                 /* legacy rlc firmware loading */
3151                 r = gfx_v9_0_rlc_load_microcode(adev);
3152                 if (r)
3153                         return r;
3154         }
3155
3156         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3157         case IP_VERSION(9, 2, 2):
3158         case IP_VERSION(9, 1, 0):
3159                 gfx_v9_0_init_lbpw(adev);
3160                 if (amdgpu_lbpw == 0)
3161                         gfx_v9_0_enable_lbpw(adev, false);
3162                 else
3163                         gfx_v9_0_enable_lbpw(adev, true);
3164                 break;
3165         case IP_VERSION(9, 4, 0):
3166                 gfx_v9_4_init_lbpw(adev);
3167                 if (amdgpu_lbpw > 0)
3168                         gfx_v9_0_enable_lbpw(adev, true);
3169                 else
3170                         gfx_v9_0_enable_lbpw(adev, false);
3171                 break;
3172         default:
3173                 break;
3174         }
3175
3176         gfx_v9_0_update_spm_vmid_internal(adev, 0xf);
3177
3178         adev->gfx.rlc.funcs->start(adev);
3179
3180         return 0;
3181 }
3182
3183 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3184 {
3185         u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3186
3187         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3188         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3189         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3190         WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3191         udelay(50);
3192 }
3193
3194 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3195 {
3196         const struct gfx_firmware_header_v1_0 *pfp_hdr;
3197         const struct gfx_firmware_header_v1_0 *ce_hdr;
3198         const struct gfx_firmware_header_v1_0 *me_hdr;
3199         const __le32 *fw_data;
3200         unsigned i, fw_size;
3201
3202         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3203                 return -EINVAL;
3204
3205         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3206                 adev->gfx.pfp_fw->data;
3207         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3208                 adev->gfx.ce_fw->data;
3209         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3210                 adev->gfx.me_fw->data;
3211
3212         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3213         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3214         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3215
3216         gfx_v9_0_cp_gfx_enable(adev, false);
3217
3218         /* PFP */
3219         fw_data = (const __le32 *)
3220                 (adev->gfx.pfp_fw->data +
3221                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3222         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3223         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3224         for (i = 0; i < fw_size; i++)
3225                 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3226         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3227
3228         /* CE */
3229         fw_data = (const __le32 *)
3230                 (adev->gfx.ce_fw->data +
3231                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3232         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3233         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3234         for (i = 0; i < fw_size; i++)
3235                 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3236         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3237
3238         /* ME */
3239         fw_data = (const __le32 *)
3240                 (adev->gfx.me_fw->data +
3241                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3242         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3243         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3244         for (i = 0; i < fw_size; i++)
3245                 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3246         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3247
3248         return 0;
3249 }
3250
3251 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3252 {
3253         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3254         const struct cs_section_def *sect = NULL;
3255         const struct cs_extent_def *ext = NULL;
3256         int r, i, tmp;
3257
3258         /* init the CP */
3259         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3260         WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3261
3262         gfx_v9_0_cp_gfx_enable(adev, true);
3263
3264         /* Now only limit the quirk on the APU gfx9 series and already
3265          * confirmed that the APU gfx10/gfx11 needn't such update.
3266          */
3267         if (adev->flags & AMD_IS_APU &&
3268                         adev->in_s3 && !adev->suspend_complete) {
3269                 DRM_INFO(" Will skip the CSB packet resubmit\n");
3270                 return 0;
3271         }
3272         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3273         if (r) {
3274                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3275                 return r;
3276         }
3277
3278         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3279         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3280
3281         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3282         amdgpu_ring_write(ring, 0x80000000);
3283         amdgpu_ring_write(ring, 0x80000000);
3284
3285         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3286                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3287                         if (sect->id == SECT_CONTEXT) {
3288                                 amdgpu_ring_write(ring,
3289                                        PACKET3(PACKET3_SET_CONTEXT_REG,
3290                                                ext->reg_count));
3291                                 amdgpu_ring_write(ring,
3292                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3293                                 for (i = 0; i < ext->reg_count; i++)
3294                                         amdgpu_ring_write(ring, ext->extent[i]);
3295                         }
3296                 }
3297         }
3298
3299         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3300         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3301
3302         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3303         amdgpu_ring_write(ring, 0);
3304
3305         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3306         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3307         amdgpu_ring_write(ring, 0x8000);
3308         amdgpu_ring_write(ring, 0x8000);
3309
3310         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3311         tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3312                 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3313         amdgpu_ring_write(ring, tmp);
3314         amdgpu_ring_write(ring, 0);
3315
3316         amdgpu_ring_commit(ring);
3317
3318         return 0;
3319 }
3320
3321 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3322 {
3323         struct amdgpu_ring *ring;
3324         u32 tmp;
3325         u32 rb_bufsz;
3326         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3327
3328         /* Set the write pointer delay */
3329         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3330
3331         /* set the RB to use vmid 0 */
3332         WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3333
3334         /* Set ring buffer size */
3335         ring = &adev->gfx.gfx_ring[0];
3336         rb_bufsz = order_base_2(ring->ring_size / 8);
3337         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3338         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3339 #ifdef __BIG_ENDIAN
3340         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3341 #endif
3342         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3343
3344         /* Initialize the ring buffer's write pointers */
3345         ring->wptr = 0;
3346         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3347         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3348
3349         /* set the wb address wether it's enabled or not */
3350         rptr_addr = ring->rptr_gpu_addr;
3351         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3352         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3353
3354         wptr_gpu_addr = ring->wptr_gpu_addr;
3355         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3356         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3357
3358         mdelay(1);
3359         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3360
3361         rb_addr = ring->gpu_addr >> 8;
3362         WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3363         WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3364
3365         tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3366         if (ring->use_doorbell) {
3367                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3368                                     DOORBELL_OFFSET, ring->doorbell_index);
3369                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3370                                     DOORBELL_EN, 1);
3371         } else {
3372                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3373         }
3374         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3375
3376         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3377                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
3378         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3379
3380         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3381                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3382
3383
3384         /* start the ring */
3385         gfx_v9_0_cp_gfx_start(adev);
3386
3387         return 0;
3388 }
3389
3390 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3391 {
3392         if (enable) {
3393                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3394         } else {
3395                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3396                         (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3397                 adev->gfx.kiq[0].ring.sched.ready = false;
3398         }
3399         udelay(50);
3400 }
3401
3402 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3403 {
3404         const struct gfx_firmware_header_v1_0 *mec_hdr;
3405         const __le32 *fw_data;
3406         unsigned i;
3407         u32 tmp;
3408
3409         if (!adev->gfx.mec_fw)
3410                 return -EINVAL;
3411
3412         gfx_v9_0_cp_compute_enable(adev, false);
3413
3414         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3415         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3416
3417         fw_data = (const __le32 *)
3418                 (adev->gfx.mec_fw->data +
3419                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3420         tmp = 0;
3421         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3422         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3423         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3424
3425         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3426                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3427         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3428                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3429
3430         /* MEC1 */
3431         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3432                          mec_hdr->jt_offset);
3433         for (i = 0; i < mec_hdr->jt_size; i++)
3434                 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3435                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3436
3437         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3438                         adev->gfx.mec_fw_version);
3439         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3440
3441         return 0;
3442 }
3443
3444 /* KIQ functions */
3445 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3446 {
3447         uint32_t tmp;
3448         struct amdgpu_device *adev = ring->adev;
3449
3450         /* tell RLC which is KIQ queue */
3451         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3452         tmp &= 0xffffff00;
3453         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3454         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3455         tmp |= 0x80;
3456         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3457 }
3458
3459 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3460 {
3461         struct amdgpu_device *adev = ring->adev;
3462
3463         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3464                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3465                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3466                         mqd->cp_hqd_queue_priority =
3467                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3468                 }
3469         }
3470 }
3471
3472 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3473 {
3474         struct amdgpu_device *adev = ring->adev;
3475         struct v9_mqd *mqd = ring->mqd_ptr;
3476         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3477         uint32_t tmp;
3478
3479         mqd->header = 0xC0310800;
3480         mqd->compute_pipelinestat_enable = 0x00000001;
3481         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3482         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3483         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3484         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3485         mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3486         mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3487         mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3488         mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3489         mqd->compute_misc_reserved = 0x00000003;
3490
3491         mqd->dynamic_cu_mask_addr_lo =
3492                 lower_32_bits(ring->mqd_gpu_addr
3493                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3494         mqd->dynamic_cu_mask_addr_hi =
3495                 upper_32_bits(ring->mqd_gpu_addr
3496                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3497
3498         eop_base_addr = ring->eop_gpu_addr >> 8;
3499         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3500         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3501
3502         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3503         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3504         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3505                         (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3506
3507         mqd->cp_hqd_eop_control = tmp;
3508
3509         /* enable doorbell? */
3510         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3511
3512         if (ring->use_doorbell) {
3513                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3514                                     DOORBELL_OFFSET, ring->doorbell_index);
3515                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3516                                     DOORBELL_EN, 1);
3517                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3518                                     DOORBELL_SOURCE, 0);
3519                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3520                                     DOORBELL_HIT, 0);
3521         } else {
3522                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3523                                          DOORBELL_EN, 0);
3524         }
3525
3526         mqd->cp_hqd_pq_doorbell_control = tmp;
3527
3528         /* disable the queue if it's active */
3529         ring->wptr = 0;
3530         mqd->cp_hqd_dequeue_request = 0;
3531         mqd->cp_hqd_pq_rptr = 0;
3532         mqd->cp_hqd_pq_wptr_lo = 0;
3533         mqd->cp_hqd_pq_wptr_hi = 0;
3534
3535         /* set the pointer to the MQD */
3536         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3537         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3538
3539         /* set MQD vmid to 0 */
3540         tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3541         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3542         mqd->cp_mqd_control = tmp;
3543
3544         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3545         hqd_gpu_addr = ring->gpu_addr >> 8;
3546         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3547         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3548
3549         /* set up the HQD, this is similar to CP_RB0_CNTL */
3550         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3551         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3552                             (order_base_2(ring->ring_size / 4) - 1));
3553         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3554                         (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3555 #ifdef __BIG_ENDIAN
3556         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3557 #endif
3558         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3559         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3560         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3561         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3562         mqd->cp_hqd_pq_control = tmp;
3563
3564         /* set the wb address whether it's enabled or not */
3565         wb_gpu_addr = ring->rptr_gpu_addr;
3566         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3567         mqd->cp_hqd_pq_rptr_report_addr_hi =
3568                 upper_32_bits(wb_gpu_addr) & 0xffff;
3569
3570         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3571         wb_gpu_addr = ring->wptr_gpu_addr;
3572         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3573         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3574
3575         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3576         ring->wptr = 0;
3577         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3578
3579         /* set the vmid for the queue */
3580         mqd->cp_hqd_vmid = 0;
3581
3582         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3583         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3584         mqd->cp_hqd_persistent_state = tmp;
3585
3586         /* set MIN_IB_AVAIL_SIZE */
3587         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3588         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3589         mqd->cp_hqd_ib_control = tmp;
3590
3591         /* set static priority for a queue/ring */
3592         gfx_v9_0_mqd_set_priority(ring, mqd);
3593         mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
3594
3595         /* map_queues packet doesn't need activate the queue,
3596          * so only kiq need set this field.
3597          */
3598         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3599                 mqd->cp_hqd_active = 1;
3600
3601         return 0;
3602 }
3603
3604 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3605 {
3606         struct amdgpu_device *adev = ring->adev;
3607         struct v9_mqd *mqd = ring->mqd_ptr;
3608         int j;
3609
3610         /* disable wptr polling */
3611         WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3612
3613         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3614                mqd->cp_hqd_eop_base_addr_lo);
3615         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3616                mqd->cp_hqd_eop_base_addr_hi);
3617
3618         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3619         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3620                mqd->cp_hqd_eop_control);
3621
3622         /* enable doorbell? */
3623         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3624                mqd->cp_hqd_pq_doorbell_control);
3625
3626         /* disable the queue if it's active */
3627         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3628                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3629                 for (j = 0; j < adev->usec_timeout; j++) {
3630                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3631                                 break;
3632                         udelay(1);
3633                 }
3634                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3635                        mqd->cp_hqd_dequeue_request);
3636                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3637                        mqd->cp_hqd_pq_rptr);
3638                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3639                        mqd->cp_hqd_pq_wptr_lo);
3640                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3641                        mqd->cp_hqd_pq_wptr_hi);
3642         }
3643
3644         /* set the pointer to the MQD */
3645         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3646                mqd->cp_mqd_base_addr_lo);
3647         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3648                mqd->cp_mqd_base_addr_hi);
3649
3650         /* set MQD vmid to 0 */
3651         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3652                mqd->cp_mqd_control);
3653
3654         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3655         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3656                mqd->cp_hqd_pq_base_lo);
3657         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3658                mqd->cp_hqd_pq_base_hi);
3659
3660         /* set up the HQD, this is similar to CP_RB0_CNTL */
3661         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3662                mqd->cp_hqd_pq_control);
3663
3664         /* set the wb address whether it's enabled or not */
3665         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3666                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3667         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3668                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3669
3670         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3671         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3672                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3673         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3674                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3675
3676         /* enable the doorbell if requested */
3677         if (ring->use_doorbell) {
3678                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3679                                         (adev->doorbell_index.kiq * 2) << 2);
3680                 /* If GC has entered CGPG, ringing doorbell > first page
3681                  * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3682                  * workaround this issue. And this change has to align with firmware
3683                  * update.
3684                  */
3685                 if (check_if_enlarge_doorbell_range(adev))
3686                         WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3687                                         (adev->doorbell.size - 4));
3688                 else
3689                         WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3690                                         (adev->doorbell_index.userqueue_end * 2) << 2);
3691         }
3692
3693         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3694                mqd->cp_hqd_pq_doorbell_control);
3695
3696         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3697         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3698                mqd->cp_hqd_pq_wptr_lo);
3699         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3700                mqd->cp_hqd_pq_wptr_hi);
3701
3702         /* set the vmid for the queue */
3703         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3704
3705         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3706                mqd->cp_hqd_persistent_state);
3707
3708         /* activate the queue */
3709         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3710                mqd->cp_hqd_active);
3711
3712         if (ring->use_doorbell)
3713                 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3714
3715         return 0;
3716 }
3717
3718 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3719 {
3720         struct amdgpu_device *adev = ring->adev;
3721         int j;
3722
3723         /* disable the queue if it's active */
3724         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3725
3726                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3727
3728                 for (j = 0; j < adev->usec_timeout; j++) {
3729                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3730                                 break;
3731                         udelay(1);
3732                 }
3733
3734                 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3735                         DRM_DEBUG("KIQ dequeue request failed.\n");
3736
3737                         /* Manual disable if dequeue request times out */
3738                         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3739                 }
3740
3741                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3742                       0);
3743         }
3744
3745         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3746         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3747         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3748         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3749         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3750         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3751         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3752         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3753
3754         return 0;
3755 }
3756
3757 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3758 {
3759         struct amdgpu_device *adev = ring->adev;
3760         struct v9_mqd *mqd = ring->mqd_ptr;
3761         struct v9_mqd *tmp_mqd;
3762
3763         gfx_v9_0_kiq_setting(ring);
3764
3765         /* GPU could be in bad state during probe, driver trigger the reset
3766          * after load the SMU, in this case , the mqd is not be initialized.
3767          * driver need to re-init the mqd.
3768          * check mqd->cp_hqd_pq_control since this value should not be 0
3769          */
3770         tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup;
3771         if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3772                 /* for GPU_RESET case , reset MQD to a clean status */
3773                 if (adev->gfx.kiq[0].mqd_backup)
3774                         memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation));
3775
3776                 /* reset ring buffer */
3777                 ring->wptr = 0;
3778                 amdgpu_ring_clear_ring(ring);
3779
3780                 mutex_lock(&adev->srbm_mutex);
3781                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3782                 gfx_v9_0_kiq_init_register(ring);
3783                 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3784                 mutex_unlock(&adev->srbm_mutex);
3785         } else {
3786                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3787                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3788                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3789                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3790                         amdgpu_ring_clear_ring(ring);
3791                 mutex_lock(&adev->srbm_mutex);
3792                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3793                 gfx_v9_0_mqd_init(ring);
3794                 gfx_v9_0_kiq_init_register(ring);
3795                 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3796                 mutex_unlock(&adev->srbm_mutex);
3797
3798                 if (adev->gfx.kiq[0].mqd_backup)
3799                         memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));
3800         }
3801
3802         return 0;
3803 }
3804
3805 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring, bool restore)
3806 {
3807         struct amdgpu_device *adev = ring->adev;
3808         struct v9_mqd *mqd = ring->mqd_ptr;
3809         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3810         struct v9_mqd *tmp_mqd;
3811
3812         /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3813          * is not be initialized before
3814          */
3815         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3816
3817         if (!restore && (!tmp_mqd->cp_hqd_pq_control ||
3818             (!amdgpu_in_reset(adev) && !adev->in_suspend))) {
3819                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3820                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3821                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3822                 mutex_lock(&adev->srbm_mutex);
3823                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3824                 gfx_v9_0_mqd_init(ring);
3825                 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3826                 mutex_unlock(&adev->srbm_mutex);
3827
3828                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3829                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3830         } else {
3831                 /* restore MQD to a clean status */
3832                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3833                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3834                 /* reset ring buffer */
3835                 ring->wptr = 0;
3836                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
3837                 amdgpu_ring_clear_ring(ring);
3838         }
3839
3840         return 0;
3841 }
3842
3843 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3844 {
3845         struct amdgpu_ring *ring;
3846         int r;
3847
3848         ring = &adev->gfx.kiq[0].ring;
3849
3850         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3851         if (unlikely(r != 0))
3852                 return r;
3853
3854         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3855         if (unlikely(r != 0)) {
3856                 amdgpu_bo_unreserve(ring->mqd_obj);
3857                 return r;
3858         }
3859
3860         gfx_v9_0_kiq_init_queue(ring);
3861         amdgpu_bo_kunmap(ring->mqd_obj);
3862         ring->mqd_ptr = NULL;
3863         amdgpu_bo_unreserve(ring->mqd_obj);
3864         return 0;
3865 }
3866
3867 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3868 {
3869         struct amdgpu_ring *ring = NULL;
3870         int r = 0, i;
3871
3872         gfx_v9_0_cp_compute_enable(adev, true);
3873
3874         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3875                 ring = &adev->gfx.compute_ring[i];
3876
3877                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3878                 if (unlikely(r != 0))
3879                         goto done;
3880                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3881                 if (!r) {
3882                         r = gfx_v9_0_kcq_init_queue(ring, false);
3883                         amdgpu_bo_kunmap(ring->mqd_obj);
3884                         ring->mqd_ptr = NULL;
3885                 }
3886                 amdgpu_bo_unreserve(ring->mqd_obj);
3887                 if (r)
3888                         goto done;
3889         }
3890
3891         r = amdgpu_gfx_enable_kcq(adev, 0);
3892 done:
3893         return r;
3894 }
3895
3896 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3897 {
3898         int r, i;
3899         struct amdgpu_ring *ring;
3900
3901         if (!(adev->flags & AMD_IS_APU))
3902                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3903
3904         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3905                 if (adev->gfx.num_gfx_rings) {
3906                         /* legacy firmware loading */
3907                         r = gfx_v9_0_cp_gfx_load_microcode(adev);
3908                         if (r)
3909                                 return r;
3910                 }
3911
3912                 r = gfx_v9_0_cp_compute_load_microcode(adev);
3913                 if (r)
3914                         return r;
3915         }
3916
3917         r = gfx_v9_0_kiq_resume(adev);
3918         if (r)
3919                 return r;
3920
3921         if (adev->gfx.num_gfx_rings) {
3922                 r = gfx_v9_0_cp_gfx_resume(adev);
3923                 if (r)
3924                         return r;
3925         }
3926
3927         r = gfx_v9_0_kcq_resume(adev);
3928         if (r)
3929                 return r;
3930
3931         if (adev->gfx.num_gfx_rings) {
3932                 ring = &adev->gfx.gfx_ring[0];
3933                 r = amdgpu_ring_test_helper(ring);
3934                 if (r)
3935                         return r;
3936         }
3937
3938         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3939                 ring = &adev->gfx.compute_ring[i];
3940                 amdgpu_ring_test_helper(ring);
3941         }
3942
3943         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3944
3945         return 0;
3946 }
3947
3948 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3949 {
3950         u32 tmp;
3951
3952         if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) &&
3953             amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))
3954                 return;
3955
3956         tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3957         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3958                                 adev->df.hash_status.hash_64k);
3959         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3960                                 adev->df.hash_status.hash_2m);
3961         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3962                                 adev->df.hash_status.hash_1g);
3963         WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3964 }
3965
3966 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3967 {
3968         if (adev->gfx.num_gfx_rings)
3969                 gfx_v9_0_cp_gfx_enable(adev, enable);
3970         gfx_v9_0_cp_compute_enable(adev, enable);
3971 }
3972
3973 static int gfx_v9_0_hw_init(void *handle)
3974 {
3975         int r;
3976         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3977
3978         amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
3979                                        adev->gfx.cleaner_shader_ptr);
3980
3981         if (!amdgpu_sriov_vf(adev))
3982                 gfx_v9_0_init_golden_registers(adev);
3983
3984         gfx_v9_0_constants_init(adev);
3985
3986         gfx_v9_0_init_tcp_config(adev);
3987
3988         r = adev->gfx.rlc.funcs->resume(adev);
3989         if (r)
3990                 return r;
3991
3992         r = gfx_v9_0_cp_resume(adev);
3993         if (r)
3994                 return r;
3995
3996         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
3997                 gfx_v9_4_2_set_power_brake_sequence(adev);
3998
3999         return r;
4000 }
4001
4002 static int gfx_v9_0_hw_fini(void *handle)
4003 {
4004         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4005
4006         if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4007                 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4008         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4009         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4010         amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
4011
4012         /* DF freeze and kcq disable will fail */
4013         if (!amdgpu_ras_intr_triggered())
4014                 /* disable KCQ to avoid CPC touch memory not valid anymore */
4015                 amdgpu_gfx_disable_kcq(adev, 0);
4016
4017         if (amdgpu_sriov_vf(adev)) {
4018                 gfx_v9_0_cp_gfx_enable(adev, false);
4019                 /* must disable polling for SRIOV when hw finished, otherwise
4020                  * CPC engine may still keep fetching WB address which is already
4021                  * invalid after sw finished and trigger DMAR reading error in
4022                  * hypervisor side.
4023                  */
4024                 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4025                 return 0;
4026         }
4027
4028         /* Use deinitialize sequence from CAIL when unbinding device from driver,
4029          * otherwise KIQ is hanging when binding back
4030          */
4031         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4032                 mutex_lock(&adev->srbm_mutex);
4033                 soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me,
4034                                 adev->gfx.kiq[0].ring.pipe,
4035                                 adev->gfx.kiq[0].ring.queue, 0, 0);
4036                 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring);
4037                 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
4038                 mutex_unlock(&adev->srbm_mutex);
4039         }
4040
4041         gfx_v9_0_cp_enable(adev, false);
4042
4043         /* Skip stopping RLC with A+A reset or when RLC controls GFX clock */
4044         if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||
4045             (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) {
4046                 dev_dbg(adev->dev, "Skipping RLC halt\n");
4047                 return 0;
4048         }
4049
4050         adev->gfx.rlc.funcs->stop(adev);
4051         return 0;
4052 }
4053
4054 static int gfx_v9_0_suspend(void *handle)
4055 {
4056         return gfx_v9_0_hw_fini(handle);
4057 }
4058
4059 static int gfx_v9_0_resume(void *handle)
4060 {
4061         return gfx_v9_0_hw_init(handle);
4062 }
4063
4064 static bool gfx_v9_0_is_idle(void *handle)
4065 {
4066         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4067
4068         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4069                                 GRBM_STATUS, GUI_ACTIVE))
4070                 return false;
4071         else
4072                 return true;
4073 }
4074
4075 static int gfx_v9_0_wait_for_idle(void *handle)
4076 {
4077         unsigned i;
4078         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4079
4080         for (i = 0; i < adev->usec_timeout; i++) {
4081                 if (gfx_v9_0_is_idle(handle))
4082                         return 0;
4083                 udelay(1);
4084         }
4085         return -ETIMEDOUT;
4086 }
4087
4088 static int gfx_v9_0_soft_reset(void *handle)
4089 {
4090         u32 grbm_soft_reset = 0;
4091         u32 tmp;
4092         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4093
4094         /* GRBM_STATUS */
4095         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4096         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4097                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4098                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4099                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4100                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4101                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4102                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4103                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4104                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4105                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4106         }
4107
4108         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4109                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4110                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4111         }
4112
4113         /* GRBM_STATUS2 */
4114         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4115         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4116                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4117                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4118
4119
4120         if (grbm_soft_reset) {
4121                 /* stop the rlc */
4122                 adev->gfx.rlc.funcs->stop(adev);
4123
4124                 if (adev->gfx.num_gfx_rings)
4125                         /* Disable GFX parsing/prefetching */
4126                         gfx_v9_0_cp_gfx_enable(adev, false);
4127
4128                 /* Disable MEC parsing/prefetching */
4129                 gfx_v9_0_cp_compute_enable(adev, false);
4130
4131                 if (grbm_soft_reset) {
4132                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4133                         tmp |= grbm_soft_reset;
4134                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4135                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4136                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4137
4138                         udelay(50);
4139
4140                         tmp &= ~grbm_soft_reset;
4141                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4142                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4143                 }
4144
4145                 /* Wait a little for things to settle down */
4146                 udelay(50);
4147         }
4148         return 0;
4149 }
4150
4151 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4152 {
4153         signed long r, cnt = 0;
4154         unsigned long flags;
4155         uint32_t seq, reg_val_offs = 0;
4156         uint64_t value = 0;
4157         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
4158         struct amdgpu_ring *ring = &kiq->ring;
4159
4160         BUG_ON(!ring->funcs->emit_rreg);
4161
4162         spin_lock_irqsave(&kiq->ring_lock, flags);
4163         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
4164                 pr_err("critical bug! too many kiq readers\n");
4165                 goto failed_unlock;
4166         }
4167         amdgpu_ring_alloc(ring, 32);
4168         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4169         amdgpu_ring_write(ring, 9 |     /* src: register*/
4170                                 (5 << 8) |      /* dst: memory */
4171                                 (1 << 16) |     /* count sel */
4172                                 (1 << 20));     /* write confirm */
4173         amdgpu_ring_write(ring, 0);
4174         amdgpu_ring_write(ring, 0);
4175         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4176                                 reg_val_offs * 4));
4177         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4178                                 reg_val_offs * 4));
4179         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4180         if (r)
4181                 goto failed_undo;
4182
4183         amdgpu_ring_commit(ring);
4184         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4185
4186         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4187
4188         /* don't wait anymore for gpu reset case because this way may
4189          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4190          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4191          * never return if we keep waiting in virt_kiq_rreg, which cause
4192          * gpu_recover() hang there.
4193          *
4194          * also don't wait anymore for IRQ context
4195          * */
4196         if (r < 1 && (amdgpu_in_reset(adev)))
4197                 goto failed_kiq_read;
4198
4199         might_sleep();
4200         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4201                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4202                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4203         }
4204
4205         if (cnt > MAX_KIQ_REG_TRY)
4206                 goto failed_kiq_read;
4207
4208         mb();
4209         value = (uint64_t)adev->wb.wb[reg_val_offs] |
4210                 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4211         amdgpu_device_wb_free(adev, reg_val_offs);
4212         return value;
4213
4214 failed_undo:
4215         amdgpu_ring_undo(ring);
4216 failed_unlock:
4217         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4218 failed_kiq_read:
4219         if (reg_val_offs)
4220                 amdgpu_device_wb_free(adev, reg_val_offs);
4221         pr_err("failed to read gpu clock\n");
4222         return ~0;
4223 }
4224
4225 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4226 {
4227         uint64_t clock, clock_lo, clock_hi, hi_check;
4228
4229         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
4230         case IP_VERSION(9, 3, 0):
4231                 preempt_disable();
4232                 clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4233                 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4234                 hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4235                 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
4236                  * roughly every 42 seconds.
4237                  */
4238                 if (hi_check != clock_hi) {
4239                         clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4240                         clock_hi = hi_check;
4241                 }
4242                 preempt_enable();
4243                 clock = clock_lo | (clock_hi << 32ULL);
4244                 break;
4245         default:
4246                 amdgpu_gfx_off_ctrl(adev, false);
4247                 mutex_lock(&adev->gfx.gpu_clock_mutex);
4248                 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
4249                             IP_VERSION(9, 0, 1) &&
4250                     amdgpu_sriov_runtime(adev)) {
4251                         clock = gfx_v9_0_kiq_read_clock(adev);
4252                 } else {
4253                         WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4254                         clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4255                                 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4256                 }
4257                 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4258                 amdgpu_gfx_off_ctrl(adev, true);
4259                 break;
4260         }
4261         return clock;
4262 }
4263
4264 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4265                                           uint32_t vmid,
4266                                           uint32_t gds_base, uint32_t gds_size,
4267                                           uint32_t gws_base, uint32_t gws_size,
4268                                           uint32_t oa_base, uint32_t oa_size)
4269 {
4270         struct amdgpu_device *adev = ring->adev;
4271
4272         /* GDS Base */
4273         gfx_v9_0_write_data_to_reg(ring, 0, false,
4274                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4275                                    gds_base);
4276
4277         /* GDS Size */
4278         gfx_v9_0_write_data_to_reg(ring, 0, false,
4279                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4280                                    gds_size);
4281
4282         /* GWS */
4283         gfx_v9_0_write_data_to_reg(ring, 0, false,
4284                                    SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4285                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4286
4287         /* OA */
4288         gfx_v9_0_write_data_to_reg(ring, 0, false,
4289                                    SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4290                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
4291 }
4292
4293 static const u32 vgpr_init_compute_shader[] =
4294 {
4295         0xb07c0000, 0xbe8000ff,
4296         0x000000f8, 0xbf110800,
4297         0x7e000280, 0x7e020280,
4298         0x7e040280, 0x7e060280,
4299         0x7e080280, 0x7e0a0280,
4300         0x7e0c0280, 0x7e0e0280,
4301         0x80808800, 0xbe803200,
4302         0xbf84fff5, 0xbf9c0000,
4303         0xd28c0001, 0x0001007f,
4304         0xd28d0001, 0x0002027e,
4305         0x10020288, 0xb8810904,
4306         0xb7814000, 0xd1196a01,
4307         0x00000301, 0xbe800087,
4308         0xbefc00c1, 0xd89c4000,
4309         0x00020201, 0xd89cc080,
4310         0x00040401, 0x320202ff,
4311         0x00000800, 0x80808100,
4312         0xbf84fff8, 0x7e020280,
4313         0xbf810000, 0x00000000,
4314 };
4315
4316 static const u32 sgpr_init_compute_shader[] =
4317 {
4318         0xb07c0000, 0xbe8000ff,
4319         0x0000005f, 0xbee50080,
4320         0xbe812c65, 0xbe822c65,
4321         0xbe832c65, 0xbe842c65,
4322         0xbe852c65, 0xb77c0005,
4323         0x80808500, 0xbf84fff8,
4324         0xbe800080, 0xbf810000,
4325 };
4326
4327 static const u32 vgpr_init_compute_shader_arcturus[] = {
4328         0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4329         0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4330         0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4331         0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4332         0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4333         0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4334         0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4335         0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4336         0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4337         0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4338         0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4339         0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4340         0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4341         0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4342         0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4343         0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4344         0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4345         0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4346         0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4347         0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4348         0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4349         0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4350         0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4351         0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4352         0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4353         0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4354         0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4355         0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4356         0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4357         0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4358         0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4359         0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4360         0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4361         0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4362         0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4363         0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4364         0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4365         0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4366         0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4367         0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4368         0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4369         0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4370         0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4371         0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4372         0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4373         0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4374         0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4375         0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4376         0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4377         0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4378         0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4379         0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4380         0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4381         0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4382         0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4383         0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4384         0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4385         0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4386         0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4387         0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4388         0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4389         0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4390         0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4391         0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4392         0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4393         0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4394         0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4395         0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4396         0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4397         0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4398         0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4399         0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4400         0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4401         0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4402         0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4403         0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4404         0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4405         0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4406         0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4407         0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4408         0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4409         0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4410         0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4411         0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4412         0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4413         0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4414         0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4415         0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4416         0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4417         0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4418         0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4419         0xbf84fff8, 0xbf810000,
4420 };
4421
4422 /* When below register arrays changed, please update gpr_reg_size,
4423   and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4424   to cover all gfx9 ASICs */
4425 static const struct soc15_reg_entry vgpr_init_regs[] = {
4426    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4427    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4428    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4429    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4430    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4431    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4432    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4433    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4434    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4435    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4436    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4437    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4438    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4439    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4440 };
4441
4442 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4443    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4444    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4445    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4446    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4447    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4448    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4449    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4450    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4451    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4452    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4453    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4454    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4455    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4456    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4457 };
4458
4459 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4460    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4461    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4462    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4463    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4464    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4465    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4466    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4467    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4468    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4469    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4470    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4471    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4472    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4473    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4474 };
4475
4476 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4477    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4478    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4479    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4480    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4481    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4482    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4483    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4484    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4485    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4486    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4487    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4488    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4489    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4490    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4491 };
4492
4493 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4494    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4495    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4496    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4497    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4498    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4499    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4500    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4501    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4502    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4503    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4504    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4505    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4506    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4507    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4508    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4509    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4510    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4511    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4512    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4513    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4514    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4515    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4516    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4517    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4518    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4519    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4520    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4521    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4522    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4523    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4524    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4525    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4526    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4527 };
4528
4529 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4530 {
4531         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4532         int i, r;
4533
4534         /* only support when RAS is enabled */
4535         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4536                 return 0;
4537
4538         r = amdgpu_ring_alloc(ring, 7);
4539         if (r) {
4540                 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4541                         ring->name, r);
4542                 return r;
4543         }
4544
4545         WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4546         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4547
4548         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4549         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4550                                 PACKET3_DMA_DATA_DST_SEL(1) |
4551                                 PACKET3_DMA_DATA_SRC_SEL(2) |
4552                                 PACKET3_DMA_DATA_ENGINE(0)));
4553         amdgpu_ring_write(ring, 0);
4554         amdgpu_ring_write(ring, 0);
4555         amdgpu_ring_write(ring, 0);
4556         amdgpu_ring_write(ring, 0);
4557         amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4558                                 adev->gds.gds_size);
4559
4560         amdgpu_ring_commit(ring);
4561
4562         for (i = 0; i < adev->usec_timeout; i++) {
4563                 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4564                         break;
4565                 udelay(1);
4566         }
4567
4568         if (i >= adev->usec_timeout)
4569                 r = -ETIMEDOUT;
4570
4571         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4572
4573         return r;
4574 }
4575
4576 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4577 {
4578         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4579         struct amdgpu_ib ib;
4580         struct dma_fence *f = NULL;
4581         int r, i;
4582         unsigned total_size, vgpr_offset, sgpr_offset;
4583         u64 gpu_addr;
4584
4585         int compute_dim_x = adev->gfx.config.max_shader_engines *
4586                                                 adev->gfx.config.max_cu_per_sh *
4587                                                 adev->gfx.config.max_sh_per_se;
4588         int sgpr_work_group_size = 5;
4589         int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4590         int vgpr_init_shader_size;
4591         const u32 *vgpr_init_shader_ptr;
4592         const struct soc15_reg_entry *vgpr_init_regs_ptr;
4593
4594         /* only support when RAS is enabled */
4595         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4596                 return 0;
4597
4598         /* bail if the compute ring is not ready */
4599         if (!ring->sched.ready)
4600                 return 0;
4601
4602         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) {
4603                 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4604                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4605                 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4606         } else {
4607                 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4608                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4609                 vgpr_init_regs_ptr = vgpr_init_regs;
4610         }
4611
4612         total_size =
4613                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4614         total_size +=
4615                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4616         total_size +=
4617                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4618         total_size = ALIGN(total_size, 256);
4619         vgpr_offset = total_size;
4620         total_size += ALIGN(vgpr_init_shader_size, 256);
4621         sgpr_offset = total_size;
4622         total_size += sizeof(sgpr_init_compute_shader);
4623
4624         /* allocate an indirect buffer to put the commands in */
4625         memset(&ib, 0, sizeof(ib));
4626         r = amdgpu_ib_get(adev, NULL, total_size,
4627                                         AMDGPU_IB_POOL_DIRECT, &ib);
4628         if (r) {
4629                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4630                 return r;
4631         }
4632
4633         /* load the compute shaders */
4634         for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4635                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4636
4637         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4638                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4639
4640         /* init the ib length to 0 */
4641         ib.length_dw = 0;
4642
4643         /* VGPR */
4644         /* write the register state for the compute dispatch */
4645         for (i = 0; i < gpr_reg_size; i++) {
4646                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4647                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4648                                                                 - PACKET3_SET_SH_REG_START;
4649                 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4650         }
4651         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4652         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4653         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4654         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4655                                                         - PACKET3_SET_SH_REG_START;
4656         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4657         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4658
4659         /* write dispatch packet */
4660         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4661         ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4662         ib.ptr[ib.length_dw++] = 1; /* y */
4663         ib.ptr[ib.length_dw++] = 1; /* z */
4664         ib.ptr[ib.length_dw++] =
4665                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4666
4667         /* write CS partial flush packet */
4668         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4669         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4670
4671         /* SGPR1 */
4672         /* write the register state for the compute dispatch */
4673         for (i = 0; i < gpr_reg_size; i++) {
4674                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4675                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4676                                                                 - PACKET3_SET_SH_REG_START;
4677                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4678         }
4679         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4680         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4681         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4682         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4683                                                         - PACKET3_SET_SH_REG_START;
4684         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4685         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4686
4687         /* write dispatch packet */
4688         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4689         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4690         ib.ptr[ib.length_dw++] = 1; /* y */
4691         ib.ptr[ib.length_dw++] = 1; /* z */
4692         ib.ptr[ib.length_dw++] =
4693                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4694
4695         /* write CS partial flush packet */
4696         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4697         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4698
4699         /* SGPR2 */
4700         /* write the register state for the compute dispatch */
4701         for (i = 0; i < gpr_reg_size; i++) {
4702                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4703                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4704                                                                 - PACKET3_SET_SH_REG_START;
4705                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4706         }
4707         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4708         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4709         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4710         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4711                                                         - PACKET3_SET_SH_REG_START;
4712         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4713         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4714
4715         /* write dispatch packet */
4716         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4717         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4718         ib.ptr[ib.length_dw++] = 1; /* y */
4719         ib.ptr[ib.length_dw++] = 1; /* z */
4720         ib.ptr[ib.length_dw++] =
4721                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4722
4723         /* write CS partial flush packet */
4724         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4725         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4726
4727         /* shedule the ib on the ring */
4728         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4729         if (r) {
4730                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4731                 goto fail;
4732         }
4733
4734         /* wait for the GPU to finish processing the IB */
4735         r = dma_fence_wait(f, false);
4736         if (r) {
4737                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4738                 goto fail;
4739         }
4740
4741 fail:
4742         amdgpu_ib_free(adev, &ib, NULL);
4743         dma_fence_put(f);
4744
4745         return r;
4746 }
4747
4748 static int gfx_v9_0_early_init(void *handle)
4749 {
4750         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4751
4752         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
4753
4754         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
4755             amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4756                 adev->gfx.num_gfx_rings = 0;
4757         else
4758                 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4759         adev->gfx.xcc_mask = 1;
4760         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4761                                           AMDGPU_MAX_COMPUTE_RINGS);
4762         gfx_v9_0_set_kiq_pm4_funcs(adev);
4763         gfx_v9_0_set_ring_funcs(adev);
4764         gfx_v9_0_set_irq_funcs(adev);
4765         gfx_v9_0_set_gds_init(adev);
4766         gfx_v9_0_set_rlc_funcs(adev);
4767
4768         /* init rlcg reg access ctrl */
4769         gfx_v9_0_init_rlcg_reg_access_ctrl(adev);
4770
4771         return gfx_v9_0_init_microcode(adev);
4772 }
4773
4774 static int gfx_v9_0_ecc_late_init(void *handle)
4775 {
4776         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4777         int r;
4778
4779         /*
4780          * Temp workaround to fix the issue that CP firmware fails to
4781          * update read pointer when CPDMA is writing clearing operation
4782          * to GDS in suspend/resume sequence on several cards. So just
4783          * limit this operation in cold boot sequence.
4784          */
4785         if ((!adev->in_suspend) &&
4786             (adev->gds.gds_size)) {
4787                 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4788                 if (r)
4789                         return r;
4790         }
4791
4792         /* requires IBs so do in late init after IB pool is initialized */
4793         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4794                 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4795         else
4796                 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4797
4798         if (r)
4799                 return r;
4800
4801         if (adev->gfx.ras &&
4802             adev->gfx.ras->enable_watchdog_timer)
4803                 adev->gfx.ras->enable_watchdog_timer(adev);
4804
4805         return 0;
4806 }
4807
4808 static int gfx_v9_0_late_init(void *handle)
4809 {
4810         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4811         int r;
4812
4813         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4814         if (r)
4815                 return r;
4816
4817         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4818         if (r)
4819                 return r;
4820
4821         r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
4822         if (r)
4823                 return r;
4824
4825         r = gfx_v9_0_ecc_late_init(handle);
4826         if (r)
4827                 return r;
4828
4829         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4830                 gfx_v9_4_2_debug_trap_config_init(adev,
4831                         adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4832         else
4833                 gfx_v9_0_debug_trap_config_init(adev,
4834                         adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4835
4836         return 0;
4837 }
4838
4839 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4840 {
4841         uint32_t rlc_setting;
4842
4843         /* if RLC is not enabled, do nothing */
4844         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4845         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4846                 return false;
4847
4848         return true;
4849 }
4850
4851 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
4852 {
4853         uint32_t data;
4854         unsigned i;
4855
4856         data = RLC_SAFE_MODE__CMD_MASK;
4857         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4858         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4859
4860         /* wait for RLC_SAFE_MODE */
4861         for (i = 0; i < adev->usec_timeout; i++) {
4862                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4863                         break;
4864                 udelay(1);
4865         }
4866 }
4867
4868 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
4869 {
4870         uint32_t data;
4871
4872         data = RLC_SAFE_MODE__CMD_MASK;
4873         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4874 }
4875
4876 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4877                                                 bool enable)
4878 {
4879         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4880
4881         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4882                 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4883                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4884                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4885         } else {
4886                 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4887                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4888                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4889         }
4890
4891         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4892 }
4893
4894 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4895                                                 bool enable)
4896 {
4897         /* TODO: double check if we need to perform under safe mode */
4898         /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4899
4900         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4901                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4902         else
4903                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4904
4905         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4906                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4907         else
4908                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4909
4910         /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4911 }
4912
4913 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4914                                                       bool enable)
4915 {
4916         uint32_t data, def;
4917
4918         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4919
4920         /* It is disabled by HW by default */
4921         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4922                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4923                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4924
4925                 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
4926                         data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4927
4928                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4929                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4930                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4931
4932                 /* only for Vega10 & Raven1 */
4933                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4934
4935                 if (def != data)
4936                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4937
4938                 /* MGLS is a global flag to control all MGLS in GFX */
4939                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4940                         /* 2 - RLC memory Light sleep */
4941                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4942                                 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4943                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4944                                 if (def != data)
4945                                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4946                         }
4947                         /* 3 - CP memory Light sleep */
4948                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4949                                 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4950                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4951                                 if (def != data)
4952                                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4953                         }
4954                 }
4955         } else {
4956                 /* 1 - MGCG_OVERRIDE */
4957                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4958
4959                 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
4960                         data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4961
4962                 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4963                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4964                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4965                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4966
4967                 if (def != data)
4968                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4969
4970                 /* 2 - disable MGLS in RLC */
4971                 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4972                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4973                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4974                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4975                 }
4976
4977                 /* 3 - disable MGLS in CP */
4978                 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4979                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4980                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4981                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4982                 }
4983         }
4984
4985         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4986 }
4987
4988 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4989                                            bool enable)
4990 {
4991         uint32_t data, def;
4992
4993         if (!adev->gfx.num_gfx_rings)
4994                 return;
4995
4996         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4997
4998         /* Enable 3D CGCG/CGLS */
4999         if (enable) {
5000                 /* write cmd to clear cgcg/cgls ov */
5001                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5002                 /* unset CGCG override */
5003                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5004                 /* update CGCG and CGLS override bits */
5005                 if (def != data)
5006                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5007
5008                 /* enable 3Dcgcg FSM(0x0000363f) */
5009                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5010
5011                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5012                         data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5013                                 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5014                 else
5015                         data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
5016
5017                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5018                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5019                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5020                 if (def != data)
5021                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5022
5023                 /* set IDLE_POLL_COUNT(0x00900100) */
5024                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5025                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5026                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5027                 if (def != data)
5028                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5029         } else {
5030                 /* Disable CGCG/CGLS */
5031                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5032                 /* disable cgcg, cgls should be disabled */
5033                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
5034                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
5035                 /* disable cgcg and cgls in FSM */
5036                 if (def != data)
5037                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5038         }
5039
5040         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5041 }
5042
5043 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5044                                                       bool enable)
5045 {
5046         uint32_t def, data;
5047
5048         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5049
5050         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5051                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5052                 /* unset CGCG override */
5053                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5054                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5055                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5056                 else
5057                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5058                 /* update CGCG and CGLS override bits */
5059                 if (def != data)
5060                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5061
5062                 /* enable cgcg FSM(0x0000363F) */
5063                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5064
5065                 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1))
5066                         data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5067                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5068                 else
5069                         data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5070                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5071                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5072                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5073                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5074                 if (def != data)
5075                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5076
5077                 /* set IDLE_POLL_COUNT(0x00900100) */
5078                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5079                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5080                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5081                 if (def != data)
5082                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5083         } else {
5084                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5085                 /* reset CGCG/CGLS bits */
5086                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5087                 /* disable cgcg and cgls in FSM */
5088                 if (def != data)
5089                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5090         }
5091
5092         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5093 }
5094
5095 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5096                                             bool enable)
5097 {
5098         if (enable) {
5099                 /* CGCG/CGLS should be enabled after MGCG/MGLS
5100                  * ===  MGCG + MGLS ===
5101                  */
5102                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5103                 /* ===  CGCG /CGLS for GFX 3D Only === */
5104                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5105                 /* ===  CGCG + CGLS === */
5106                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5107         } else {
5108                 /* CGCG/CGLS should be disabled before MGCG/MGLS
5109                  * ===  CGCG + CGLS ===
5110                  */
5111                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5112                 /* ===  CGCG /CGLS for GFX 3D Only === */
5113                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5114                 /* ===  MGCG + MGLS === */
5115                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5116         }
5117         return 0;
5118 }
5119
5120 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
5121                                               unsigned int vmid)
5122 {
5123         u32 reg, data;
5124
5125         reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5126         if (amdgpu_sriov_is_pp_one_vf(adev))
5127                 data = RREG32_NO_KIQ(reg);
5128         else
5129                 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);
5130
5131         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5132         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5133
5134         if (amdgpu_sriov_is_pp_one_vf(adev))
5135                 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5136         else
5137                 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5138 }
5139
5140 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid)
5141 {
5142         amdgpu_gfx_off_ctrl(adev, false);
5143
5144         gfx_v9_0_update_spm_vmid_internal(adev, vmid);
5145
5146         amdgpu_gfx_off_ctrl(adev, true);
5147 }
5148
5149 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5150                                         uint32_t offset,
5151                                         struct soc15_reg_rlcg *entries, int arr_size)
5152 {
5153         int i;
5154         uint32_t reg;
5155
5156         if (!entries)
5157                 return false;
5158
5159         for (i = 0; i < arr_size; i++) {
5160                 const struct soc15_reg_rlcg *entry;
5161
5162                 entry = &entries[i];
5163                 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5164                 if (offset == reg)
5165                         return true;
5166         }
5167
5168         return false;
5169 }
5170
5171 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5172 {
5173         return gfx_v9_0_check_rlcg_range(adev, offset,
5174                                         (void *)rlcg_access_gc_9_0,
5175                                         ARRAY_SIZE(rlcg_access_gc_9_0));
5176 }
5177
5178 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5179         .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5180         .set_safe_mode = gfx_v9_0_set_safe_mode,
5181         .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5182         .init = gfx_v9_0_rlc_init,
5183         .get_csb_size = gfx_v9_0_get_csb_size,
5184         .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5185         .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5186         .resume = gfx_v9_0_rlc_resume,
5187         .stop = gfx_v9_0_rlc_stop,
5188         .reset = gfx_v9_0_rlc_reset,
5189         .start = gfx_v9_0_rlc_start,
5190         .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5191         .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5192 };
5193
5194 static int gfx_v9_0_set_powergating_state(void *handle,
5195                                           enum amd_powergating_state state)
5196 {
5197         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5198         bool enable = (state == AMD_PG_STATE_GATE);
5199
5200         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5201         case IP_VERSION(9, 2, 2):
5202         case IP_VERSION(9, 1, 0):
5203         case IP_VERSION(9, 3, 0):
5204                 if (!enable)
5205                         amdgpu_gfx_off_ctrl(adev, false);
5206
5207                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5208                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5209                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5210                 } else {
5211                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5212                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5213                 }
5214
5215                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5216                         gfx_v9_0_enable_cp_power_gating(adev, true);
5217                 else
5218                         gfx_v9_0_enable_cp_power_gating(adev, false);
5219
5220                 /* update gfx cgpg state */
5221                 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5222
5223                 /* update mgcg state */
5224                 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5225
5226                 if (enable)
5227                         amdgpu_gfx_off_ctrl(adev, true);
5228                 break;
5229         case IP_VERSION(9, 2, 1):
5230                 amdgpu_gfx_off_ctrl(adev, enable);
5231                 break;
5232         default:
5233                 break;
5234         }
5235
5236         return 0;
5237 }
5238
5239 static int gfx_v9_0_set_clockgating_state(void *handle,
5240                                           enum amd_clockgating_state state)
5241 {
5242         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5243
5244         if (amdgpu_sriov_vf(adev))
5245                 return 0;
5246
5247         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5248         case IP_VERSION(9, 0, 1):
5249         case IP_VERSION(9, 2, 1):
5250         case IP_VERSION(9, 4, 0):
5251         case IP_VERSION(9, 2, 2):
5252         case IP_VERSION(9, 1, 0):
5253         case IP_VERSION(9, 4, 1):
5254         case IP_VERSION(9, 3, 0):
5255         case IP_VERSION(9, 4, 2):
5256                 gfx_v9_0_update_gfx_clock_gating(adev,
5257                                                  state == AMD_CG_STATE_GATE);
5258                 break;
5259         default:
5260                 break;
5261         }
5262         return 0;
5263 }
5264
5265 static void gfx_v9_0_get_clockgating_state(void *handle, u64 *flags)
5266 {
5267         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5268         int data;
5269
5270         if (amdgpu_sriov_vf(adev))
5271                 *flags = 0;
5272
5273         /* AMD_CG_SUPPORT_GFX_MGCG */
5274         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5275         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5276                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5277
5278         /* AMD_CG_SUPPORT_GFX_CGCG */
5279         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5280         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5281                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5282
5283         /* AMD_CG_SUPPORT_GFX_CGLS */
5284         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5285                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5286
5287         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5288         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5289         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5290                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5291
5292         /* AMD_CG_SUPPORT_GFX_CP_LS */
5293         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5294         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5295                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5296
5297         if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) {
5298                 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5299                 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5300                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5301                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5302
5303                 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5304                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5305                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5306         }
5307 }
5308
5309 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5310 {
5311         return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/
5312 }
5313
5314 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5315 {
5316         struct amdgpu_device *adev = ring->adev;
5317         u64 wptr;
5318
5319         /* XXX check if swapping is necessary on BE */
5320         if (ring->use_doorbell) {
5321                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5322         } else {
5323                 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5324                 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5325         }
5326
5327         return wptr;
5328 }
5329
5330 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5331 {
5332         struct amdgpu_device *adev = ring->adev;
5333
5334         if (ring->use_doorbell) {
5335                 /* XXX check if swapping is necessary on BE */
5336                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5337                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5338         } else {
5339                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5340                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5341         }
5342 }
5343
5344 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5345 {
5346         struct amdgpu_device *adev = ring->adev;
5347         u32 ref_and_mask, reg_mem_engine;
5348         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5349
5350         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5351                 switch (ring->me) {
5352                 case 1:
5353                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5354                         break;
5355                 case 2:
5356                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5357                         break;
5358                 default:
5359                         return;
5360                 }
5361                 reg_mem_engine = 0;
5362         } else {
5363                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5364                 reg_mem_engine = 1; /* pfp */
5365         }
5366
5367         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5368                               adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5369                               adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5370                               ref_and_mask, ref_and_mask, 0x20);
5371 }
5372
5373 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5374                                         struct amdgpu_job *job,
5375                                         struct amdgpu_ib *ib,
5376                                         uint32_t flags)
5377 {
5378         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5379         u32 header, control = 0;
5380
5381         if (ib->flags & AMDGPU_IB_FLAG_CE)
5382                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5383         else
5384                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5385
5386         control |= ib->length_dw | (vmid << 24);
5387
5388         if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
5389                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5390
5391                 if (flags & AMDGPU_IB_PREEMPTED)
5392                         control |= INDIRECT_BUFFER_PRE_RESUME(1);
5393
5394                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5395                         gfx_v9_0_ring_emit_de_meta(ring,
5396                                                    (!amdgpu_sriov_vf(ring->adev) &&
5397                                                    flags & AMDGPU_IB_PREEMPTED) ?
5398                                                    true : false,
5399                                                    job->gds_size > 0 && job->gds_base != 0);
5400         }
5401
5402         amdgpu_ring_write(ring, header);
5403         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5404         amdgpu_ring_write(ring,
5405 #ifdef __BIG_ENDIAN
5406                 (2 << 0) |
5407 #endif
5408                 lower_32_bits(ib->gpu_addr));
5409         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5410         amdgpu_ring_ib_on_emit_cntl(ring);
5411         amdgpu_ring_write(ring, control);
5412 }
5413
5414 static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,
5415                                      unsigned offset)
5416 {
5417         u32 control = ring->ring[offset];
5418
5419         control |= INDIRECT_BUFFER_PRE_RESUME(1);
5420         ring->ring[offset] = control;
5421 }
5422
5423 static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,
5424                                         unsigned offset)
5425 {
5426         struct amdgpu_device *adev = ring->adev;
5427         void *ce_payload_cpu_addr;
5428         uint64_t payload_offset, payload_size;
5429
5430         payload_size = sizeof(struct v9_ce_ib_state);
5431
5432         if (ring->is_mes_queue) {
5433                 payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5434                                           gfx[0].gfx_meta_data) +
5435                         offsetof(struct v9_gfx_meta_data, ce_payload);
5436                 ce_payload_cpu_addr =
5437                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);
5438         } else {
5439                 payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5440                 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5441         }
5442
5443         if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5444                 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size);
5445         } else {
5446                 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr,
5447                        (ring->buf_mask + 1 - offset) << 2);
5448                 payload_size -= (ring->buf_mask + 1 - offset) << 2;
5449                 memcpy((void *)&ring->ring[0],
5450                        ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5451                        payload_size);
5452         }
5453 }
5454
5455 static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,
5456                                         unsigned offset)
5457 {
5458         struct amdgpu_device *adev = ring->adev;
5459         void *de_payload_cpu_addr;
5460         uint64_t payload_offset, payload_size;
5461
5462         payload_size = sizeof(struct v9_de_ib_state);
5463
5464         if (ring->is_mes_queue) {
5465                 payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5466                                           gfx[0].gfx_meta_data) +
5467                         offsetof(struct v9_gfx_meta_data, de_payload);
5468                 de_payload_cpu_addr =
5469                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);
5470         } else {
5471                 payload_offset = offsetof(struct v9_gfx_meta_data, de_payload);
5472                 de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5473         }
5474
5475         ((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status =
5476                 IB_COMPLETION_STATUS_PREEMPTED;
5477
5478         if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5479                 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size);
5480         } else {
5481                 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr,
5482                        (ring->buf_mask + 1 - offset) << 2);
5483                 payload_size -= (ring->buf_mask + 1 - offset) << 2;
5484                 memcpy((void *)&ring->ring[0],
5485                        de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5486                        payload_size);
5487         }
5488 }
5489
5490 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5491                                           struct amdgpu_job *job,
5492                                           struct amdgpu_ib *ib,
5493                                           uint32_t flags)
5494 {
5495         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5496         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5497
5498         /* Currently, there is a high possibility to get wave ID mismatch
5499          * between ME and GDS, leading to a hw deadlock, because ME generates
5500          * different wave IDs than the GDS expects. This situation happens
5501          * randomly when at least 5 compute pipes use GDS ordered append.
5502          * The wave IDs generated by ME are also wrong after suspend/resume.
5503          * Those are probably bugs somewhere else in the kernel driver.
5504          *
5505          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5506          * GDS to 0 for this ring (me/pipe).
5507          */
5508         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5509                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5510                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5511                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5512         }
5513
5514         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5515         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5516         amdgpu_ring_write(ring,
5517 #ifdef __BIG_ENDIAN
5518                                 (2 << 0) |
5519 #endif
5520                                 lower_32_bits(ib->gpu_addr));
5521         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5522         amdgpu_ring_write(ring, control);
5523 }
5524
5525 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5526                                      u64 seq, unsigned flags)
5527 {
5528         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5529         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5530         bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5531         bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;
5532         uint32_t dw2 = 0;
5533
5534         /* RELEASE_MEM - flush caches, send int */
5535         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5536
5537         if (writeback) {
5538                 dw2 = EOP_TC_NC_ACTION_EN;
5539         } else {
5540                 dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN |
5541                                 EOP_TC_MD_ACTION_EN;
5542         }
5543         dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5544                                 EVENT_INDEX(5);
5545         if (exec)
5546                 dw2 |= EOP_EXEC;
5547
5548         amdgpu_ring_write(ring, dw2);
5549         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5550
5551         /*
5552          * the address should be Qword aligned if 64bit write, Dword
5553          * aligned if only send 32bit data low (discard data high)
5554          */
5555         if (write64bit)
5556                 BUG_ON(addr & 0x7);
5557         else
5558                 BUG_ON(addr & 0x3);
5559         amdgpu_ring_write(ring, lower_32_bits(addr));
5560         amdgpu_ring_write(ring, upper_32_bits(addr));
5561         amdgpu_ring_write(ring, lower_32_bits(seq));
5562         amdgpu_ring_write(ring, upper_32_bits(seq));
5563         amdgpu_ring_write(ring, 0);
5564 }
5565
5566 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5567 {
5568         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5569         uint32_t seq = ring->fence_drv.sync_seq;
5570         uint64_t addr = ring->fence_drv.gpu_addr;
5571
5572         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5573                               lower_32_bits(addr), upper_32_bits(addr),
5574                               seq, 0xffffffff, 4);
5575 }
5576
5577 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5578                                         unsigned vmid, uint64_t pd_addr)
5579 {
5580         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5581
5582         /* compute doesn't have PFP */
5583         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5584                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5585                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5586                 amdgpu_ring_write(ring, 0x0);
5587         }
5588 }
5589
5590 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5591 {
5592         return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */
5593 }
5594
5595 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5596 {
5597         u64 wptr;
5598
5599         /* XXX check if swapping is necessary on BE */
5600         if (ring->use_doorbell)
5601                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5602         else
5603                 BUG();
5604         return wptr;
5605 }
5606
5607 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5608 {
5609         struct amdgpu_device *adev = ring->adev;
5610
5611         /* XXX check if swapping is necessary on BE */
5612         if (ring->use_doorbell) {
5613                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5614                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5615         } else{
5616                 BUG(); /* only DOORBELL method supported on gfx9 now */
5617         }
5618 }
5619
5620 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5621                                          u64 seq, unsigned int flags)
5622 {
5623         struct amdgpu_device *adev = ring->adev;
5624
5625         /* we only allocate 32bit for each seq wb address */
5626         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5627
5628         /* write fence seq to the "addr" */
5629         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5630         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5631                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5632         amdgpu_ring_write(ring, lower_32_bits(addr));
5633         amdgpu_ring_write(ring, upper_32_bits(addr));
5634         amdgpu_ring_write(ring, lower_32_bits(seq));
5635
5636         if (flags & AMDGPU_FENCE_FLAG_INT) {
5637                 /* set register to trigger INT */
5638                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5639                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5640                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5641                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5642                 amdgpu_ring_write(ring, 0);
5643                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5644         }
5645 }
5646
5647 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5648 {
5649         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5650         amdgpu_ring_write(ring, 0);
5651 }
5652
5653 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
5654 {
5655         struct amdgpu_device *adev = ring->adev;
5656         struct v9_ce_ib_state ce_payload = {0};
5657         uint64_t offset, ce_payload_gpu_addr;
5658         void *ce_payload_cpu_addr;
5659         int cnt;
5660
5661         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5662
5663         if (ring->is_mes_queue) {
5664                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5665                                   gfx[0].gfx_meta_data) +
5666                         offsetof(struct v9_gfx_meta_data, ce_payload);
5667                 ce_payload_gpu_addr =
5668                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5669                 ce_payload_cpu_addr =
5670                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5671         } else {
5672                 offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5673                 ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5674                 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5675         }
5676
5677         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5678         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5679                                  WRITE_DATA_DST_SEL(8) |
5680                                  WR_CONFIRM) |
5681                                  WRITE_DATA_CACHE_POLICY(0));
5682         amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));
5683         amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));
5684
5685         amdgpu_ring_ib_on_emit_ce(ring);
5686
5687         if (resume)
5688                 amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr,
5689                                            sizeof(ce_payload) >> 2);
5690         else
5691                 amdgpu_ring_write_multiple(ring, (void *)&ce_payload,
5692                                            sizeof(ce_payload) >> 2);
5693 }
5694
5695 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)
5696 {
5697         int i, r = 0;
5698         struct amdgpu_device *adev = ring->adev;
5699         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5700         struct amdgpu_ring *kiq_ring = &kiq->ring;
5701         unsigned long flags;
5702
5703         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5704                 return -EINVAL;
5705
5706         spin_lock_irqsave(&kiq->ring_lock, flags);
5707
5708         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5709                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5710                 return -ENOMEM;
5711         }
5712
5713         /* assert preemption condition */
5714         amdgpu_ring_set_preempt_cond_exec(ring, false);
5715
5716         ring->trail_seq += 1;
5717         amdgpu_ring_alloc(ring, 13);
5718         gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
5719                                  ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);
5720
5721         /* assert IB preemption, emit the trailing fence */
5722         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5723                                    ring->trail_fence_gpu_addr,
5724                                    ring->trail_seq);
5725
5726         amdgpu_ring_commit(kiq_ring);
5727         spin_unlock_irqrestore(&kiq->ring_lock, flags);
5728
5729         /* poll the trailing fence */
5730         for (i = 0; i < adev->usec_timeout; i++) {
5731                 if (ring->trail_seq ==
5732                         le32_to_cpu(*ring->trail_fence_cpu_addr))
5733                         break;
5734                 udelay(1);
5735         }
5736
5737         if (i >= adev->usec_timeout) {
5738                 r = -EINVAL;
5739                 DRM_WARN("ring %d timeout to preempt ib\n", ring->idx);
5740         }
5741
5742         /*reset the CP_VMID_PREEMPT after trailing fence*/
5743         amdgpu_ring_emit_wreg(ring,
5744                               SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT),
5745                               0x0);
5746         amdgpu_ring_commit(ring);
5747
5748         /* deassert preemption condition */
5749         amdgpu_ring_set_preempt_cond_exec(ring, true);
5750         return r;
5751 }
5752
5753 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)
5754 {
5755         struct amdgpu_device *adev = ring->adev;
5756         struct v9_de_ib_state de_payload = {0};
5757         uint64_t offset, gds_addr, de_payload_gpu_addr;
5758         void *de_payload_cpu_addr;
5759         int cnt;
5760
5761         if (ring->is_mes_queue) {
5762                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5763                                   gfx[0].gfx_meta_data) +
5764                         offsetof(struct v9_gfx_meta_data, de_payload);
5765                 de_payload_gpu_addr =
5766                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5767                 de_payload_cpu_addr =
5768                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5769
5770                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5771                                   gfx[0].gds_backup) +
5772                         offsetof(struct v9_gfx_meta_data, de_payload);
5773                 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5774         } else {
5775                 offset = offsetof(struct v9_gfx_meta_data, de_payload);
5776                 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5777                 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5778
5779                 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5780                                  AMDGPU_CSA_SIZE - adev->gds.gds_size,
5781                                  PAGE_SIZE);
5782         }
5783
5784         if (usegds) {
5785                 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5786                 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5787         }
5788
5789         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5790         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5791         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5792                                  WRITE_DATA_DST_SEL(8) |
5793                                  WR_CONFIRM) |
5794                                  WRITE_DATA_CACHE_POLICY(0));
5795         amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5796         amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5797
5798         amdgpu_ring_ib_on_emit_de(ring);
5799         if (resume)
5800                 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5801                                            sizeof(de_payload) >> 2);
5802         else
5803                 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5804                                            sizeof(de_payload) >> 2);
5805 }
5806
5807 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5808                                    bool secure)
5809 {
5810         uint32_t v = secure ? FRAME_TMZ : 0;
5811
5812         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5813         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5814 }
5815
5816 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5817 {
5818         uint32_t dw2 = 0;
5819
5820         gfx_v9_0_ring_emit_ce_meta(ring,
5821                                    (!amdgpu_sriov_vf(ring->adev) &&
5822                                    flags & AMDGPU_IB_PREEMPTED) ? true : false);
5823
5824         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5825         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5826                 /* set load_global_config & load_global_uconfig */
5827                 dw2 |= 0x8001;
5828                 /* set load_cs_sh_regs */
5829                 dw2 |= 0x01000000;
5830                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5831                 dw2 |= 0x10002;
5832
5833                 /* set load_ce_ram if preamble presented */
5834                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5835                         dw2 |= 0x10000000;
5836         } else {
5837                 /* still load_ce_ram if this is the first time preamble presented
5838                  * although there is no context switch happens.
5839                  */
5840                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5841                         dw2 |= 0x10000000;
5842         }
5843
5844         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5845         amdgpu_ring_write(ring, dw2);
5846         amdgpu_ring_write(ring, 0);
5847 }
5848
5849 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5850                                                   uint64_t addr)
5851 {
5852         unsigned ret;
5853         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5854         amdgpu_ring_write(ring, lower_32_bits(addr));
5855         amdgpu_ring_write(ring, upper_32_bits(addr));
5856         /* discard following DWs if *cond_exec_gpu_addr==0 */
5857         amdgpu_ring_write(ring, 0);
5858         ret = ring->wptr & ring->buf_mask;
5859         /* patch dummy value later */
5860         amdgpu_ring_write(ring, 0);
5861         return ret;
5862 }
5863
5864 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5865                                     uint32_t reg_val_offs)
5866 {
5867         struct amdgpu_device *adev = ring->adev;
5868
5869         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5870         amdgpu_ring_write(ring, 0 |     /* src: register*/
5871                                 (5 << 8) |      /* dst: memory */
5872                                 (1 << 20));     /* write confirm */
5873         amdgpu_ring_write(ring, reg);
5874         amdgpu_ring_write(ring, 0);
5875         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5876                                 reg_val_offs * 4));
5877         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5878                                 reg_val_offs * 4));
5879 }
5880
5881 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5882                                     uint32_t val)
5883 {
5884         uint32_t cmd = 0;
5885
5886         switch (ring->funcs->type) {
5887         case AMDGPU_RING_TYPE_GFX:
5888                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5889                 break;
5890         case AMDGPU_RING_TYPE_KIQ:
5891                 cmd = (1 << 16); /* no inc addr */
5892                 break;
5893         default:
5894                 cmd = WR_CONFIRM;
5895                 break;
5896         }
5897         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5898         amdgpu_ring_write(ring, cmd);
5899         amdgpu_ring_write(ring, reg);
5900         amdgpu_ring_write(ring, 0);
5901         amdgpu_ring_write(ring, val);
5902 }
5903
5904 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5905                                         uint32_t val, uint32_t mask)
5906 {
5907         gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5908 }
5909
5910 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5911                                                   uint32_t reg0, uint32_t reg1,
5912                                                   uint32_t ref, uint32_t mask)
5913 {
5914         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5915         struct amdgpu_device *adev = ring->adev;
5916         bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5917                 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5918
5919         if (fw_version_ok)
5920                 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5921                                       ref, mask, 0x20);
5922         else
5923                 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5924                                                            ref, mask);
5925 }
5926
5927 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5928 {
5929         struct amdgpu_device *adev = ring->adev;
5930         uint32_t value = 0;
5931
5932         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5933         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5934         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5935         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5936         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5937         WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5938         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5939 }
5940
5941 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5942                                                  enum amdgpu_interrupt_state state)
5943 {
5944         switch (state) {
5945         case AMDGPU_IRQ_STATE_DISABLE:
5946         case AMDGPU_IRQ_STATE_ENABLE:
5947                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5948                                TIME_STAMP_INT_ENABLE,
5949                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5950                 break;
5951         default:
5952                 break;
5953         }
5954 }
5955
5956 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5957                                                      int me, int pipe,
5958                                                      enum amdgpu_interrupt_state state)
5959 {
5960         u32 mec_int_cntl, mec_int_cntl_reg;
5961
5962         /*
5963          * amdgpu controls only the first MEC. That's why this function only
5964          * handles the setting of interrupts for this specific MEC. All other
5965          * pipes' interrupts are set by amdkfd.
5966          */
5967
5968         if (me == 1) {
5969                 switch (pipe) {
5970                 case 0:
5971                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5972                         break;
5973                 case 1:
5974                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5975                         break;
5976                 case 2:
5977                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5978                         break;
5979                 case 3:
5980                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5981                         break;
5982                 default:
5983                         DRM_DEBUG("invalid pipe %d\n", pipe);
5984                         return;
5985                 }
5986         } else {
5987                 DRM_DEBUG("invalid me %d\n", me);
5988                 return;
5989         }
5990
5991         switch (state) {
5992         case AMDGPU_IRQ_STATE_DISABLE:
5993                 mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg);
5994                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5995                                              TIME_STAMP_INT_ENABLE, 0);
5996                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5997                 break;
5998         case AMDGPU_IRQ_STATE_ENABLE:
5999                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6000                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6001                                              TIME_STAMP_INT_ENABLE, 1);
6002                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6003                 break;
6004         default:
6005                 break;
6006         }
6007 }
6008
6009 static u32 gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device *adev,
6010                                      int me, int pipe)
6011 {
6012         /*
6013          * amdgpu controls only the first MEC. That's why this function only
6014          * handles the setting of interrupts for this specific MEC. All other
6015          * pipes' interrupts are set by amdkfd.
6016          */
6017         if (me != 1)
6018                 return 0;
6019
6020         switch (pipe) {
6021         case 0:
6022                 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
6023         case 1:
6024                 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
6025         case 2:
6026                 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
6027         case 3:
6028                 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
6029         default:
6030                 return 0;
6031         }
6032 }
6033
6034 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6035                                              struct amdgpu_irq_src *source,
6036                                              unsigned type,
6037                                              enum amdgpu_interrupt_state state)
6038 {
6039         u32 cp_int_cntl_reg, cp_int_cntl;
6040         int i, j;
6041
6042         switch (state) {
6043         case AMDGPU_IRQ_STATE_DISABLE:
6044         case AMDGPU_IRQ_STATE_ENABLE:
6045                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6046                                PRIV_REG_INT_ENABLE,
6047                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6048                 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6049                         for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6050                                 /* MECs start at 1 */
6051                                 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6052
6053                                 if (cp_int_cntl_reg) {
6054                                         cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6055                                         cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6056                                                                     PRIV_REG_INT_ENABLE,
6057                                                                     state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6058                                         WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6059                                 }
6060                         }
6061                 }
6062                 break;
6063         default:
6064                 break;
6065         }
6066
6067         return 0;
6068 }
6069
6070 static int gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6071                                            struct amdgpu_irq_src *source,
6072                                            unsigned type,
6073                                            enum amdgpu_interrupt_state state)
6074 {
6075         u32 cp_int_cntl_reg, cp_int_cntl;
6076         int i, j;
6077
6078         switch (state) {
6079         case AMDGPU_IRQ_STATE_DISABLE:
6080         case AMDGPU_IRQ_STATE_ENABLE:
6081                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6082                                OPCODE_ERROR_INT_ENABLE,
6083                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6084                 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6085                         for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6086                                 /* MECs start at 1 */
6087                                 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6088
6089                                 if (cp_int_cntl_reg) {
6090                                         cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6091                                         cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6092                                                                     OPCODE_ERROR_INT_ENABLE,
6093                                                                     state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6094                                         WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6095                                 }
6096                         }
6097                 }
6098                 break;
6099         default:
6100                 break;
6101         }
6102
6103         return 0;
6104 }
6105
6106 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6107                                               struct amdgpu_irq_src *source,
6108                                               unsigned type,
6109                                               enum amdgpu_interrupt_state state)
6110 {
6111         switch (state) {
6112         case AMDGPU_IRQ_STATE_DISABLE:
6113         case AMDGPU_IRQ_STATE_ENABLE:
6114                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6115                                PRIV_INSTR_INT_ENABLE,
6116                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6117                 break;
6118         default:
6119                 break;
6120         }
6121
6122         return 0;
6123 }
6124
6125 #define ENABLE_ECC_ON_ME_PIPE(me, pipe)                         \
6126         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6127                         CP_ECC_ERROR_INT_ENABLE, 1)
6128
6129 #define DISABLE_ECC_ON_ME_PIPE(me, pipe)                        \
6130         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6131                         CP_ECC_ERROR_INT_ENABLE, 0)
6132
6133 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
6134                                               struct amdgpu_irq_src *source,
6135                                               unsigned type,
6136                                               enum amdgpu_interrupt_state state)
6137 {
6138         switch (state) {
6139         case AMDGPU_IRQ_STATE_DISABLE:
6140                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6141                                 CP_ECC_ERROR_INT_ENABLE, 0);
6142                 DISABLE_ECC_ON_ME_PIPE(1, 0);
6143                 DISABLE_ECC_ON_ME_PIPE(1, 1);
6144                 DISABLE_ECC_ON_ME_PIPE(1, 2);
6145                 DISABLE_ECC_ON_ME_PIPE(1, 3);
6146                 break;
6147
6148         case AMDGPU_IRQ_STATE_ENABLE:
6149                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6150                                 CP_ECC_ERROR_INT_ENABLE, 1);
6151                 ENABLE_ECC_ON_ME_PIPE(1, 0);
6152                 ENABLE_ECC_ON_ME_PIPE(1, 1);
6153                 ENABLE_ECC_ON_ME_PIPE(1, 2);
6154                 ENABLE_ECC_ON_ME_PIPE(1, 3);
6155                 break;
6156         default:
6157                 break;
6158         }
6159
6160         return 0;
6161 }
6162
6163
6164 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6165                                             struct amdgpu_irq_src *src,
6166                                             unsigned type,
6167                                             enum amdgpu_interrupt_state state)
6168 {
6169         switch (type) {
6170         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6171                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
6172                 break;
6173         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6174                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6175                 break;
6176         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6177                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6178                 break;
6179         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6180                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6181                 break;
6182         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6183                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6184                 break;
6185         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6186                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6187                 break;
6188         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6189                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6190                 break;
6191         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6192                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6193                 break;
6194         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6195                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6196                 break;
6197         default:
6198                 break;
6199         }
6200         return 0;
6201 }
6202
6203 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
6204                             struct amdgpu_irq_src *source,
6205                             struct amdgpu_iv_entry *entry)
6206 {
6207         int i;
6208         u8 me_id, pipe_id, queue_id;
6209         struct amdgpu_ring *ring;
6210
6211         DRM_DEBUG("IH: CP EOP\n");
6212         me_id = (entry->ring_id & 0x0c) >> 2;
6213         pipe_id = (entry->ring_id & 0x03) >> 0;
6214         queue_id = (entry->ring_id & 0x70) >> 4;
6215
6216         switch (me_id) {
6217         case 0:
6218                 if (adev->gfx.num_gfx_rings) {
6219                         if (!adev->gfx.mcbp) {
6220                                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6221                         } else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) {
6222                                 /* Fence signals are handled on the software rings*/
6223                                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
6224                                         amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]);
6225                         }
6226                 }
6227                 break;
6228         case 1:
6229         case 2:
6230                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6231                         ring = &adev->gfx.compute_ring[i];
6232                         /* Per-queue interrupt is supported for MEC starting from VI.
6233                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
6234                           */
6235                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6236                                 amdgpu_fence_process(ring);
6237                 }
6238                 break;
6239         }
6240         return 0;
6241 }
6242
6243 static void gfx_v9_0_fault(struct amdgpu_device *adev,
6244                            struct amdgpu_iv_entry *entry)
6245 {
6246         u8 me_id, pipe_id, queue_id;
6247         struct amdgpu_ring *ring;
6248         int i;
6249
6250         me_id = (entry->ring_id & 0x0c) >> 2;
6251         pipe_id = (entry->ring_id & 0x03) >> 0;
6252         queue_id = (entry->ring_id & 0x70) >> 4;
6253
6254         switch (me_id) {
6255         case 0:
6256                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
6257                 break;
6258         case 1:
6259         case 2:
6260                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6261                         ring = &adev->gfx.compute_ring[i];
6262                         if (ring->me == me_id && ring->pipe == pipe_id &&
6263                             ring->queue == queue_id)
6264                                 drm_sched_fault(&ring->sched);
6265                 }
6266                 break;
6267         }
6268 }
6269
6270 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
6271                                  struct amdgpu_irq_src *source,
6272                                  struct amdgpu_iv_entry *entry)
6273 {
6274         DRM_ERROR("Illegal register access in command stream\n");
6275         gfx_v9_0_fault(adev, entry);
6276         return 0;
6277 }
6278
6279 static int gfx_v9_0_bad_op_irq(struct amdgpu_device *adev,
6280                                struct amdgpu_irq_src *source,
6281                                struct amdgpu_iv_entry *entry)
6282 {
6283         DRM_ERROR("Illegal opcode in command stream\n");
6284         gfx_v9_0_fault(adev, entry);
6285         return 0;
6286 }
6287
6288 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
6289                                   struct amdgpu_irq_src *source,
6290                                   struct amdgpu_iv_entry *entry)
6291 {
6292         DRM_ERROR("Illegal instruction in command stream\n");
6293         gfx_v9_0_fault(adev, entry);
6294         return 0;
6295 }
6296
6297
6298 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
6299         { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
6300           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
6301           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
6302         },
6303         { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
6304           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
6305           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
6306         },
6307         { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6308           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
6309           0, 0
6310         },
6311         { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6312           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
6313           0, 0
6314         },
6315         { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
6316           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
6317           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
6318         },
6319         { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6320           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
6321           0, 0
6322         },
6323         { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6324           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
6325           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
6326         },
6327         { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
6328           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
6329           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
6330         },
6331         { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6332           SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
6333           0, 0
6334         },
6335         { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6336           SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
6337           0, 0
6338         },
6339         { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6340           SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
6341           0, 0
6342         },
6343         { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6344           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
6345           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
6346         },
6347         { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6348           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
6349           0, 0
6350         },
6351         { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6352           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6353           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6354         },
6355         { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6356           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6357           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6358           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6359         },
6360         { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6361           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6362           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6363           0, 0
6364         },
6365         { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6366           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6367           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6368           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6369         },
6370         { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6371           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6372           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6373           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6374         },
6375         { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6376           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6377           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6378           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6379         },
6380         { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6381           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6382           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6383           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6384         },
6385         { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6386           SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6387           0, 0
6388         },
6389         { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6390           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6391           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6392         },
6393         { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6394           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6395           0, 0
6396         },
6397         { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6398           SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6399           0, 0
6400         },
6401         { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6402           SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6403           0, 0
6404         },
6405         { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6406           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6407           0, 0
6408         },
6409         { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6410           SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6411           0, 0
6412         },
6413         { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6414           SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6415           0, 0
6416         },
6417         { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6418           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6419           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6420         },
6421         { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6422           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6423           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6424         },
6425         { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6426           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6427           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6428         },
6429         { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6430           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6431           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6432         },
6433         { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6434           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6435           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6436         },
6437         { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6438           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6439           0, 0
6440         },
6441         { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6442           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6443           0, 0
6444         },
6445         { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6446           SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6447           0, 0
6448         },
6449         { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6450           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6451           0, 0
6452         },
6453         { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6454           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6455           0, 0
6456         },
6457         { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6458           SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6459           0, 0
6460         },
6461         { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6462           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6463           0, 0
6464         },
6465         { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6466           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6467           0, 0
6468         },
6469         { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6470           SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6471           0, 0
6472         },
6473         { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6474           SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6475           0, 0
6476         },
6477         { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6478           SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6479           0, 0
6480         },
6481         { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6482           SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6483           0, 0
6484         },
6485         { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6486           SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6487           0, 0
6488         },
6489         { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6490           SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6491           0, 0
6492         },
6493         { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6494           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6495           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6496         },
6497         { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6498           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6499           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6500         },
6501         { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6502           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6503           0, 0
6504         },
6505         { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6506           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6507           0, 0
6508         },
6509         { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6510           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6511           0, 0
6512         },
6513         { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6514           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6515           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6516         },
6517         { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6518           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6519           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6520         },
6521         { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6522           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6523           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6524         },
6525         { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6526           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6527           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6528         },
6529         { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6530           SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6531           0, 0
6532         },
6533         { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6534           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6535           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6536         },
6537         { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6538           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6539           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6540         },
6541         { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6542           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6543           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6544         },
6545         { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6546           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6547           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6548         },
6549         { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6550           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6551           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6552         },
6553         { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6554           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6555           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6556         },
6557         { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6558           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6559           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6560         },
6561         { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6562           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6563           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6564         },
6565         { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6566           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6567           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6568         },
6569         { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6570           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6571           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6572         },
6573         { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6574           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6575           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6576         },
6577         { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6578           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6579           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6580         },
6581         { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6582           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6583           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6584         },
6585         { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6586           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6587           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6588         },
6589         { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6590           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6591           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6592         },
6593         { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6594           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6595           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6596         },
6597         { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6598           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6599           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6600         },
6601         { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6602           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6603           0, 0
6604         },
6605         { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6606           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6607           0, 0
6608         },
6609         { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6610           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6611           0, 0
6612         },
6613         { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6614           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6615           0, 0
6616         },
6617         { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6618           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6619           0, 0
6620         },
6621         { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6622           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6623           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6624         },
6625         { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6626           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6627           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6628         },
6629         { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6630           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6631           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6632         },
6633         { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6634           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6635           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6636         },
6637         { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6638           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6639           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6640         },
6641         { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6642           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6643           0, 0
6644         },
6645         { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6646           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6647           0, 0
6648         },
6649         { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6650           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6651           0, 0
6652         },
6653         { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6654           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6655           0, 0
6656         },
6657         { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6658           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6659           0, 0
6660         },
6661         { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6662           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6663           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6664         },
6665         { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6666           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6667           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6668         },
6669         { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6670           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6671           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6672         },
6673         { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6674           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6675           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6676         },
6677         { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6678           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6679           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6680         },
6681         { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6682           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6683           0, 0
6684         },
6685         { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6686           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6687           0, 0
6688         },
6689         { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6690           SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6691           0, 0
6692         },
6693         { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6694           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6695           0, 0
6696         },
6697         { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6698           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6699           0, 0
6700         },
6701         { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6702           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6703           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6704         },
6705         { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6706           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6707           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6708         },
6709         { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6710           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6711           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6712         },
6713         { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6714           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6715           0, 0
6716         },
6717         { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6718           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6719           0, 0
6720         },
6721         { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6722           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6723           0, 0
6724         },
6725         { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6726           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6727           0, 0
6728         },
6729         { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6730           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6731           0, 0
6732         },
6733         { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6734           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6735           0, 0
6736         }
6737 };
6738
6739 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6740                                      void *inject_if, uint32_t instance_mask)
6741 {
6742         struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6743         int ret;
6744         struct ta_ras_trigger_error_input block_info = { 0 };
6745
6746         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6747                 return -EINVAL;
6748
6749         if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6750                 return -EINVAL;
6751
6752         if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6753                 return -EPERM;
6754
6755         if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6756               info->head.type)) {
6757                 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6758                         ras_gfx_subblocks[info->head.sub_block_index].name,
6759                         info->head.type);
6760                 return -EPERM;
6761         }
6762
6763         if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6764               info->head.type)) {
6765                 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6766                         ras_gfx_subblocks[info->head.sub_block_index].name,
6767                         info->head.type);
6768                 return -EPERM;
6769         }
6770
6771         block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6772         block_info.sub_block_index =
6773                 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6774         block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6775         block_info.address = info->address;
6776         block_info.value = info->value;
6777
6778         mutex_lock(&adev->grbm_idx_mutex);
6779         ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask);
6780         mutex_unlock(&adev->grbm_idx_mutex);
6781
6782         return ret;
6783 }
6784
6785 static const char * const vml2_mems[] = {
6786         "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6787         "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6788         "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6789         "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6790         "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6791         "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6792         "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6793         "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6794         "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6795         "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6796         "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6797         "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6798         "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6799         "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6800         "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6801         "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6802 };
6803
6804 static const char * const vml2_walker_mems[] = {
6805         "UTC_VML2_CACHE_PDE0_MEM0",
6806         "UTC_VML2_CACHE_PDE0_MEM1",
6807         "UTC_VML2_CACHE_PDE1_MEM0",
6808         "UTC_VML2_CACHE_PDE1_MEM1",
6809         "UTC_VML2_CACHE_PDE2_MEM0",
6810         "UTC_VML2_CACHE_PDE2_MEM1",
6811         "UTC_VML2_RDIF_LOG_FIFO",
6812 };
6813
6814 static const char * const atc_l2_cache_2m_mems[] = {
6815         "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6816         "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6817         "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6818         "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6819 };
6820
6821 static const char *atc_l2_cache_4k_mems[] = {
6822         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6823         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6824         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6825         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6826         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6827         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6828         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6829         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6830         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6831         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6832         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6833         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6834         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6835         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6836         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6837         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6838         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6839         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6840         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6841         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6842         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6843         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6844         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6845         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6846         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6847         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6848         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6849         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6850         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6851         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6852         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6853         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6854 };
6855
6856 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6857                                          struct ras_err_data *err_data)
6858 {
6859         uint32_t i, data;
6860         uint32_t sec_count, ded_count;
6861
6862         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6863         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6864         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6865         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6866         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6867         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6868         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6869         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6870
6871         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6872                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6873                 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6874
6875                 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6876                 if (sec_count) {
6877                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6878                                 "SEC %d\n", i, vml2_mems[i], sec_count);
6879                         err_data->ce_count += sec_count;
6880                 }
6881
6882                 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6883                 if (ded_count) {
6884                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6885                                 "DED %d\n", i, vml2_mems[i], ded_count);
6886                         err_data->ue_count += ded_count;
6887                 }
6888         }
6889
6890         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6891                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6892                 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6893
6894                 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6895                                                 SEC_COUNT);
6896                 if (sec_count) {
6897                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6898                                 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6899                         err_data->ce_count += sec_count;
6900                 }
6901
6902                 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6903                                                 DED_COUNT);
6904                 if (ded_count) {
6905                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6906                                 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6907                         err_data->ue_count += ded_count;
6908                 }
6909         }
6910
6911         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6912                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6913                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6914
6915                 sec_count = (data & 0x00006000L) >> 0xd;
6916                 if (sec_count) {
6917                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6918                                 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6919                                 sec_count);
6920                         err_data->ce_count += sec_count;
6921                 }
6922         }
6923
6924         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6925                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6926                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6927
6928                 sec_count = (data & 0x00006000L) >> 0xd;
6929                 if (sec_count) {
6930                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6931                                 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6932                                 sec_count);
6933                         err_data->ce_count += sec_count;
6934                 }
6935
6936                 ded_count = (data & 0x00018000L) >> 0xf;
6937                 if (ded_count) {
6938                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6939                                 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6940                                 ded_count);
6941                         err_data->ue_count += ded_count;
6942                 }
6943         }
6944
6945         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6946         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6947         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6948         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6949
6950         return 0;
6951 }
6952
6953 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6954         const struct soc15_reg_entry *reg,
6955         uint32_t se_id, uint32_t inst_id, uint32_t value,
6956         uint32_t *sec_count, uint32_t *ded_count)
6957 {
6958         uint32_t i;
6959         uint32_t sec_cnt, ded_cnt;
6960
6961         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6962                 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6963                         gfx_v9_0_ras_fields[i].seg != reg->seg ||
6964                         gfx_v9_0_ras_fields[i].inst != reg->inst)
6965                         continue;
6966
6967                 sec_cnt = (value &
6968                                 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6969                                 gfx_v9_0_ras_fields[i].sec_count_shift;
6970                 if (sec_cnt) {
6971                         dev_info(adev->dev, "GFX SubBlock %s, "
6972                                 "Instance[%d][%d], SEC %d\n",
6973                                 gfx_v9_0_ras_fields[i].name,
6974                                 se_id, inst_id,
6975                                 sec_cnt);
6976                         *sec_count += sec_cnt;
6977                 }
6978
6979                 ded_cnt = (value &
6980                                 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6981                                 gfx_v9_0_ras_fields[i].ded_count_shift;
6982                 if (ded_cnt) {
6983                         dev_info(adev->dev, "GFX SubBlock %s, "
6984                                 "Instance[%d][%d], DED %d\n",
6985                                 gfx_v9_0_ras_fields[i].name,
6986                                 se_id, inst_id,
6987                                 ded_cnt);
6988                         *ded_count += ded_cnt;
6989                 }
6990         }
6991
6992         return 0;
6993 }
6994
6995 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6996 {
6997         int i, j, k;
6998
6999         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
7000                 return;
7001
7002         /* read back registers to clear the counters */
7003         mutex_lock(&adev->grbm_idx_mutex);
7004         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
7005                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
7006                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7007                                 amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0);
7008                                 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7009                         }
7010                 }
7011         }
7012         WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
7013         mutex_unlock(&adev->grbm_idx_mutex);
7014
7015         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7016         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
7017         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7018         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
7019         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7020         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
7021         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7022         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
7023
7024         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
7025                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
7026                 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
7027         }
7028
7029         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
7030                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
7031                 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
7032         }
7033
7034         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
7035                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
7036                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
7037         }
7038
7039         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
7040                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
7041                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
7042         }
7043
7044         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7045         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7046         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7047         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7048 }
7049
7050 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
7051                                           void *ras_error_status)
7052 {
7053         struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
7054         uint32_t sec_count = 0, ded_count = 0;
7055         uint32_t i, j, k;
7056         uint32_t reg_value;
7057
7058         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
7059                 return;
7060
7061         err_data->ue_count = 0;
7062         err_data->ce_count = 0;
7063
7064         mutex_lock(&adev->grbm_idx_mutex);
7065
7066         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
7067                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
7068                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7069                                 amdgpu_gfx_select_se_sh(adev, j, 0, k, 0);
7070                                 reg_value =
7071                                         RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7072                                 if (reg_value)
7073                                         gfx_v9_0_ras_error_count(adev,
7074                                                 &gfx_v9_0_edc_counter_regs[i],
7075                                                 j, k, reg_value,
7076                                                 &sec_count, &ded_count);
7077                         }
7078                 }
7079         }
7080
7081         err_data->ce_count += sec_count;
7082         err_data->ue_count += ded_count;
7083
7084         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7085         mutex_unlock(&adev->grbm_idx_mutex);
7086
7087         gfx_v9_0_query_utc_edc_status(adev, err_data);
7088 }
7089
7090 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
7091 {
7092         const unsigned int cp_coher_cntl =
7093                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
7094                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
7095                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
7096                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
7097                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
7098
7099         /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
7100         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
7101         amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
7102         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
7103         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
7104         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
7105         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
7106         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
7107 }
7108
7109 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
7110                                         uint32_t pipe, bool enable)
7111 {
7112         struct amdgpu_device *adev = ring->adev;
7113         uint32_t val;
7114         uint32_t wcl_cs_reg;
7115
7116         /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
7117         val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
7118
7119         switch (pipe) {
7120         case 0:
7121                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
7122                 break;
7123         case 1:
7124                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
7125                 break;
7126         case 2:
7127                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
7128                 break;
7129         case 3:
7130                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
7131                 break;
7132         default:
7133                 DRM_DEBUG("invalid pipe %d\n", pipe);
7134                 return;
7135         }
7136
7137         amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
7138
7139 }
7140 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
7141 {
7142         struct amdgpu_device *adev = ring->adev;
7143         uint32_t val;
7144         int i;
7145
7146
7147         /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
7148          * number of gfx waves. Setting 5 bit will make sure gfx only gets
7149          * around 25% of gpu resources.
7150          */
7151         val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
7152         amdgpu_ring_emit_wreg(ring,
7153                               SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
7154                               val);
7155
7156         /* Restrict waves for normal/low priority compute queues as well
7157          * to get best QoS for high priority compute jobs.
7158          *
7159          * amdgpu controls only 1st ME(0-3 CS pipes).
7160          */
7161         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
7162                 if (i != ring->pipe)
7163                         gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
7164
7165         }
7166 }
7167
7168 static void gfx_v9_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
7169 {
7170         int i;
7171
7172         /* Header itself is a NOP packet */
7173         if (num_nop == 1) {
7174                 amdgpu_ring_write(ring, ring->funcs->nop);
7175                 return;
7176         }
7177
7178         /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
7179         amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
7180
7181         /* Header is at index 0, followed by num_nops - 1 NOP packet's */
7182         for (i = 1; i < num_nop; i++)
7183                 amdgpu_ring_write(ring, ring->funcs->nop);
7184 }
7185
7186 static int gfx_v9_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid)
7187 {
7188         struct amdgpu_device *adev = ring->adev;
7189         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7190         struct amdgpu_ring *kiq_ring = &kiq->ring;
7191         unsigned long flags;
7192         u32 tmp;
7193         int r;
7194
7195         if (amdgpu_sriov_vf(adev))
7196                 return -EINVAL;
7197
7198         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
7199                 return -EINVAL;
7200
7201         spin_lock_irqsave(&kiq->ring_lock, flags);
7202
7203         if (amdgpu_ring_alloc(kiq_ring, 5)) {
7204                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7205                 return -ENOMEM;
7206         }
7207
7208         tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
7209         gfx_v9_0_ring_emit_wreg(kiq_ring,
7210                                  SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), tmp);
7211         amdgpu_ring_commit(kiq_ring);
7212
7213         spin_unlock_irqrestore(&kiq->ring_lock, flags);
7214
7215         r = amdgpu_ring_test_ring(kiq_ring);
7216         if (r)
7217                 return r;
7218
7219         if (amdgpu_ring_alloc(ring, 7 + 7 + 5))
7220                 return -ENOMEM;
7221         gfx_v9_0_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
7222                                  ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC);
7223         gfx_v9_0_ring_emit_reg_wait(ring,
7224                                     SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0, 0xffff);
7225         gfx_v9_0_ring_emit_wreg(ring,
7226                                 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0);
7227
7228         return amdgpu_ring_test_ring(ring);
7229 }
7230
7231 static int gfx_v9_0_reset_kcq(struct amdgpu_ring *ring,
7232                               unsigned int vmid)
7233 {
7234         struct amdgpu_device *adev = ring->adev;
7235         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7236         struct amdgpu_ring *kiq_ring = &kiq->ring;
7237         unsigned long flags;
7238         int i, r;
7239
7240         if (!adev->debug_exp_resets &&
7241             !adev->gfx.num_gfx_rings)
7242                 return -EINVAL;
7243
7244         if (amdgpu_sriov_vf(adev))
7245                 return -EINVAL;
7246
7247         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
7248                 return -EINVAL;
7249
7250         spin_lock_irqsave(&kiq->ring_lock, flags);
7251
7252         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
7253                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7254                 return -ENOMEM;
7255         }
7256
7257         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
7258                                    0, 0);
7259         amdgpu_ring_commit(kiq_ring);
7260
7261         spin_unlock_irqrestore(&kiq->ring_lock, flags);
7262
7263         r = amdgpu_ring_test_ring(kiq_ring);
7264         if (r)
7265                 return r;
7266
7267         /* make sure dequeue is complete*/
7268         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
7269         mutex_lock(&adev->srbm_mutex);
7270         soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
7271         for (i = 0; i < adev->usec_timeout; i++) {
7272                 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
7273                         break;
7274                 udelay(1);
7275         }
7276         if (i >= adev->usec_timeout)
7277                 r = -ETIMEDOUT;
7278         soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7279         mutex_unlock(&adev->srbm_mutex);
7280         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
7281         if (r) {
7282                 dev_err(adev->dev, "fail to wait on hqd deactive\n");
7283                 return r;
7284         }
7285
7286         r = amdgpu_bo_reserve(ring->mqd_obj, false);
7287         if (unlikely(r != 0)){
7288                 dev_err(adev->dev, "fail to resv mqd_obj\n");
7289                 return r;
7290         }
7291         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
7292         if (!r) {
7293                 r = gfx_v9_0_kcq_init_queue(ring, true);
7294                 amdgpu_bo_kunmap(ring->mqd_obj);
7295                 ring->mqd_ptr = NULL;
7296         }
7297         amdgpu_bo_unreserve(ring->mqd_obj);
7298         if (r) {
7299                 dev_err(adev->dev, "fail to unresv mqd_obj\n");
7300                 return r;
7301         }
7302         spin_lock_irqsave(&kiq->ring_lock, flags);
7303         r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
7304         if (r) {
7305                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7306                 return -ENOMEM;
7307         }
7308         kiq->pmf->kiq_map_queues(kiq_ring, ring);
7309         amdgpu_ring_commit(kiq_ring);
7310         spin_unlock_irqrestore(&kiq->ring_lock, flags);
7311         r = amdgpu_ring_test_ring(kiq_ring);
7312         if (r) {
7313                 DRM_ERROR("fail to remap queue\n");
7314                 return r;
7315         }
7316         return amdgpu_ring_test_ring(ring);
7317 }
7318
7319 static void gfx_v9_ip_print(void *handle, struct drm_printer *p)
7320 {
7321         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
7322         uint32_t i, j, k, reg, index = 0;
7323         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7324
7325         if (!adev->gfx.ip_dump_core)
7326                 return;
7327
7328         for (i = 0; i < reg_count; i++)
7329                 drm_printf(p, "%-50s \t 0x%08x\n",
7330                            gc_reg_list_9[i].reg_name,
7331                            adev->gfx.ip_dump_core[i]);
7332
7333         /* print compute queue registers for all instances */
7334         if (!adev->gfx.ip_dump_compute_queues)
7335                 return;
7336
7337         reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7338         drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
7339                    adev->gfx.mec.num_mec,
7340                    adev->gfx.mec.num_pipe_per_mec,
7341                    adev->gfx.mec.num_queue_per_pipe);
7342
7343         for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7344                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7345                         for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7346                                 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
7347                                 for (reg = 0; reg < reg_count; reg++) {
7348                                         drm_printf(p, "%-50s \t 0x%08x\n",
7349                                                    gc_cp_reg_list_9[reg].reg_name,
7350                                                    adev->gfx.ip_dump_compute_queues[index + reg]);
7351                                 }
7352                                 index += reg_count;
7353                         }
7354                 }
7355         }
7356
7357 }
7358
7359 static void gfx_v9_ip_dump(void *handle)
7360 {
7361         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
7362         uint32_t i, j, k, reg, index = 0;
7363         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7364
7365         if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings)
7366                 return;
7367
7368         amdgpu_gfx_off_ctrl(adev, false);
7369         for (i = 0; i < reg_count; i++)
7370                 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i]));
7371         amdgpu_gfx_off_ctrl(adev, true);
7372
7373         /* dump compute queue registers for all instances */
7374         if (!adev->gfx.ip_dump_compute_queues)
7375                 return;
7376
7377         reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7378         amdgpu_gfx_off_ctrl(adev, false);
7379         mutex_lock(&adev->srbm_mutex);
7380         for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7381                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7382                         for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7383                                 /* ME0 is for GFX so start from 1 for CP */
7384                                 soc15_grbm_select(adev, 1 + i, j, k, 0, 0);
7385
7386                                 for (reg = 0; reg < reg_count; reg++) {
7387                                         adev->gfx.ip_dump_compute_queues[index + reg] =
7388                                                 RREG32(SOC15_REG_ENTRY_OFFSET(
7389                                                         gc_cp_reg_list_9[reg]));
7390                                 }
7391                                 index += reg_count;
7392                         }
7393                 }
7394         }
7395         soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7396         mutex_unlock(&adev->srbm_mutex);
7397         amdgpu_gfx_off_ctrl(adev, true);
7398
7399 }
7400
7401 static void gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
7402 {
7403         /* Emit the cleaner shader */
7404         amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
7405         amdgpu_ring_write(ring, 0);  /* RESERVED field, programmed to zero */
7406 }
7407
7408 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
7409         .name = "gfx_v9_0",
7410         .early_init = gfx_v9_0_early_init,
7411         .late_init = gfx_v9_0_late_init,
7412         .sw_init = gfx_v9_0_sw_init,
7413         .sw_fini = gfx_v9_0_sw_fini,
7414         .hw_init = gfx_v9_0_hw_init,
7415         .hw_fini = gfx_v9_0_hw_fini,
7416         .suspend = gfx_v9_0_suspend,
7417         .resume = gfx_v9_0_resume,
7418         .is_idle = gfx_v9_0_is_idle,
7419         .wait_for_idle = gfx_v9_0_wait_for_idle,
7420         .soft_reset = gfx_v9_0_soft_reset,
7421         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
7422         .set_powergating_state = gfx_v9_0_set_powergating_state,
7423         .get_clockgating_state = gfx_v9_0_get_clockgating_state,
7424         .dump_ip_state = gfx_v9_ip_dump,
7425         .print_ip_state = gfx_v9_ip_print,
7426 };
7427
7428 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
7429         .type = AMDGPU_RING_TYPE_GFX,
7430         .align_mask = 0xff,
7431         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7432         .support_64bit_ptrs = true,
7433         .secure_submission_supported = true,
7434         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
7435         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
7436         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
7437         .emit_frame_size = /* totally 242 maximum if 16 IBs */
7438                 5 +  /* COND_EXEC */
7439                 7 +  /* PIPELINE_SYNC */
7440                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7441                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7442                 2 + /* VM_FLUSH */
7443                 8 +  /* FENCE for VM_FLUSH */
7444                 20 + /* GDS switch */
7445                 4 + /* double SWITCH_BUFFER,
7446                        the first COND_EXEC jump to the place just
7447                            prior to this double SWITCH_BUFFER  */
7448                 5 + /* COND_EXEC */
7449                 7 +      /*     HDP_flush */
7450                 4 +      /*     VGT_flush */
7451                 14 + /* CE_META */
7452                 31 + /* DE_META */
7453                 3 + /* CNTX_CTRL */
7454                 5 + /* HDP_INVL */
7455                 8 + 8 + /* FENCE x2 */
7456                 2 + /* SWITCH_BUFFER */
7457                 7 + /* gfx_v9_0_emit_mem_sync */
7458                 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7459         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
7460         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7461         .emit_fence = gfx_v9_0_ring_emit_fence,
7462         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7463         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7464         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7465         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7466         .test_ring = gfx_v9_0_ring_test_ring,
7467         .insert_nop = gfx_v9_ring_insert_nop,
7468         .pad_ib = amdgpu_ring_generic_pad_ib,
7469         .emit_switch_buffer = gfx_v9_ring_emit_sb,
7470         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7471         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7472         .preempt_ib = gfx_v9_0_ring_preempt_ib,
7473         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7474         .emit_wreg = gfx_v9_0_ring_emit_wreg,
7475         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7476         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7477         .soft_recovery = gfx_v9_0_ring_soft_recovery,
7478         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7479         .reset = gfx_v9_0_reset_kgq,
7480         .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7481         .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7482         .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7483 };
7484
7485 static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {
7486         .type = AMDGPU_RING_TYPE_GFX,
7487         .align_mask = 0xff,
7488         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7489         .support_64bit_ptrs = true,
7490         .secure_submission_supported = true,
7491         .get_rptr = amdgpu_sw_ring_get_rptr_gfx,
7492         .get_wptr = amdgpu_sw_ring_get_wptr_gfx,
7493         .set_wptr = amdgpu_sw_ring_set_wptr_gfx,
7494         .emit_frame_size = /* totally 242 maximum if 16 IBs */
7495                 5 +  /* COND_EXEC */
7496                 7 +  /* PIPELINE_SYNC */
7497                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7498                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7499                 2 + /* VM_FLUSH */
7500                 8 +  /* FENCE for VM_FLUSH */
7501                 20 + /* GDS switch */
7502                 4 + /* double SWITCH_BUFFER,
7503                      * the first COND_EXEC jump to the place just
7504                      * prior to this double SWITCH_BUFFER
7505                      */
7506                 5 + /* COND_EXEC */
7507                 7 +      /*     HDP_flush */
7508                 4 +      /*     VGT_flush */
7509                 14 + /* CE_META */
7510                 31 + /* DE_META */
7511                 3 + /* CNTX_CTRL */
7512                 5 + /* HDP_INVL */
7513                 8 + 8 + /* FENCE x2 */
7514                 2 + /* SWITCH_BUFFER */
7515                 7 + /* gfx_v9_0_emit_mem_sync */
7516                 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7517         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
7518         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7519         .emit_fence = gfx_v9_0_ring_emit_fence,
7520         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7521         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7522         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7523         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7524         .test_ring = gfx_v9_0_ring_test_ring,
7525         .test_ib = gfx_v9_0_ring_test_ib,
7526         .insert_nop = gfx_v9_ring_insert_nop,
7527         .pad_ib = amdgpu_ring_generic_pad_ib,
7528         .emit_switch_buffer = gfx_v9_ring_emit_sb,
7529         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7530         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7531         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7532         .emit_wreg = gfx_v9_0_ring_emit_wreg,
7533         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7534         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7535         .soft_recovery = gfx_v9_0_ring_soft_recovery,
7536         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7537         .patch_cntl = gfx_v9_0_ring_patch_cntl,
7538         .patch_de = gfx_v9_0_ring_patch_de_meta,
7539         .patch_ce = gfx_v9_0_ring_patch_ce_meta,
7540         .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7541         .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7542         .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7543 };
7544
7545 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
7546         .type = AMDGPU_RING_TYPE_COMPUTE,
7547         .align_mask = 0xff,
7548         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7549         .support_64bit_ptrs = true,
7550         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
7551         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
7552         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
7553         .emit_frame_size =
7554                 20 + /* gfx_v9_0_ring_emit_gds_switch */
7555                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
7556                 5 + /* hdp invalidate */
7557                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7558                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7559                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7560                 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
7561                 7 + /* gfx_v9_0_emit_mem_sync */
7562                 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
7563                 15 + /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
7564                 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7565         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7566         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
7567         .emit_fence = gfx_v9_0_ring_emit_fence,
7568         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7569         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7570         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7571         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7572         .test_ring = gfx_v9_0_ring_test_ring,
7573         .test_ib = gfx_v9_0_ring_test_ib,
7574         .insert_nop = gfx_v9_ring_insert_nop,
7575         .pad_ib = amdgpu_ring_generic_pad_ib,
7576         .emit_wreg = gfx_v9_0_ring_emit_wreg,
7577         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7578         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7579         .soft_recovery = gfx_v9_0_ring_soft_recovery,
7580         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7581         .emit_wave_limit = gfx_v9_0_emit_wave_limit,
7582         .reset = gfx_v9_0_reset_kcq,
7583         .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7584         .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7585         .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7586 };
7587
7588 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
7589         .type = AMDGPU_RING_TYPE_KIQ,
7590         .align_mask = 0xff,
7591         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7592         .support_64bit_ptrs = true,
7593         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
7594         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
7595         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
7596         .emit_frame_size =
7597                 20 + /* gfx_v9_0_ring_emit_gds_switch */
7598                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
7599                 5 + /* hdp invalidate */
7600                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7601                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7602                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7603                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7604         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7605         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
7606         .test_ring = gfx_v9_0_ring_test_ring,
7607         .insert_nop = amdgpu_ring_insert_nop,
7608         .pad_ib = amdgpu_ring_generic_pad_ib,
7609         .emit_rreg = gfx_v9_0_ring_emit_rreg,
7610         .emit_wreg = gfx_v9_0_ring_emit_wreg,
7611         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7612         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7613 };
7614
7615 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
7616 {
7617         int i;
7618
7619         adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq;
7620
7621         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7622                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
7623
7624         if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
7625                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
7626                         adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;
7627         }
7628
7629         for (i = 0; i < adev->gfx.num_compute_rings; i++)
7630                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
7631 }
7632
7633 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
7634         .set = gfx_v9_0_set_eop_interrupt_state,
7635         .process = gfx_v9_0_eop_irq,
7636 };
7637
7638 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
7639         .set = gfx_v9_0_set_priv_reg_fault_state,
7640         .process = gfx_v9_0_priv_reg_irq,
7641 };
7642
7643 static const struct amdgpu_irq_src_funcs gfx_v9_0_bad_op_irq_funcs = {
7644         .set = gfx_v9_0_set_bad_op_fault_state,
7645         .process = gfx_v9_0_bad_op_irq,
7646 };
7647
7648 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
7649         .set = gfx_v9_0_set_priv_inst_fault_state,
7650         .process = gfx_v9_0_priv_inst_irq,
7651 };
7652
7653 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
7654         .set = gfx_v9_0_set_cp_ecc_error_state,
7655         .process = amdgpu_gfx_cp_ecc_error_irq,
7656 };
7657
7658
7659 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
7660 {
7661         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7662         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
7663
7664         adev->gfx.priv_reg_irq.num_types = 1;
7665         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
7666
7667         adev->gfx.bad_op_irq.num_types = 1;
7668         adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs;
7669
7670         adev->gfx.priv_inst_irq.num_types = 1;
7671         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7672
7673         adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7674         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7675 }
7676
7677 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7678 {
7679         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7680         case IP_VERSION(9, 0, 1):
7681         case IP_VERSION(9, 2, 1):
7682         case IP_VERSION(9, 4, 0):
7683         case IP_VERSION(9, 2, 2):
7684         case IP_VERSION(9, 1, 0):
7685         case IP_VERSION(9, 4, 1):
7686         case IP_VERSION(9, 3, 0):
7687         case IP_VERSION(9, 4, 2):
7688                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7689                 break;
7690         default:
7691                 break;
7692         }
7693 }
7694
7695 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7696 {
7697         /* init asci gds info */
7698         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7699         case IP_VERSION(9, 0, 1):
7700         case IP_VERSION(9, 2, 1):
7701         case IP_VERSION(9, 4, 0):
7702                 adev->gds.gds_size = 0x10000;
7703                 break;
7704         case IP_VERSION(9, 2, 2):
7705         case IP_VERSION(9, 1, 0):
7706         case IP_VERSION(9, 4, 1):
7707                 adev->gds.gds_size = 0x1000;
7708                 break;
7709         case IP_VERSION(9, 4, 2):
7710                 /* aldebaran removed all the GDS internal memory,
7711                  * only support GWS opcode in kernel, like barrier
7712                  * semaphore.etc */
7713                 adev->gds.gds_size = 0;
7714                 break;
7715         default:
7716                 adev->gds.gds_size = 0x10000;
7717                 break;
7718         }
7719
7720         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7721         case IP_VERSION(9, 0, 1):
7722         case IP_VERSION(9, 4, 0):
7723                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7724                 break;
7725         case IP_VERSION(9, 2, 1):
7726                 adev->gds.gds_compute_max_wave_id = 0x27f;
7727                 break;
7728         case IP_VERSION(9, 2, 2):
7729         case IP_VERSION(9, 1, 0):
7730                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7731                         adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7732                 else
7733                         adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7734                 break;
7735         case IP_VERSION(9, 4, 1):
7736                 adev->gds.gds_compute_max_wave_id = 0xfff;
7737                 break;
7738         case IP_VERSION(9, 4, 2):
7739                 /* deprecated for Aldebaran, no usage at all */
7740                 adev->gds.gds_compute_max_wave_id = 0;
7741                 break;
7742         default:
7743                 /* this really depends on the chip */
7744                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7745                 break;
7746         }
7747
7748         adev->gds.gws_size = 64;
7749         adev->gds.oa_size = 16;
7750 }
7751
7752 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7753                                                  u32 bitmap)
7754 {
7755         u32 data;
7756
7757         if (!bitmap)
7758                 return;
7759
7760         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7761         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7762
7763         WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7764 }
7765
7766 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7767 {
7768         u32 data, mask;
7769
7770         data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7771         data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7772
7773         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7774         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7775
7776         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7777
7778         return (~data) & mask;
7779 }
7780
7781 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7782                                  struct amdgpu_cu_info *cu_info)
7783 {
7784         int i, j, k, counter, active_cu_number = 0;
7785         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7786         unsigned disable_masks[4 * 4];
7787
7788         if (!adev || !cu_info)
7789                 return -EINVAL;
7790
7791         /*
7792          * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7793          */
7794         if (adev->gfx.config.max_shader_engines *
7795                 adev->gfx.config.max_sh_per_se > 16)
7796                 return -EINVAL;
7797
7798         amdgpu_gfx_parse_disable_cu(disable_masks,
7799                                     adev->gfx.config.max_shader_engines,
7800                                     adev->gfx.config.max_sh_per_se);
7801
7802         mutex_lock(&adev->grbm_idx_mutex);
7803         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7804                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7805                         mask = 1;
7806                         ao_bitmap = 0;
7807                         counter = 0;
7808                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
7809                         gfx_v9_0_set_user_cu_inactive_bitmap(
7810                                 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7811                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7812
7813                         /*
7814                          * The bitmap(and ao_cu_bitmap) in cu_info structure is
7815                          * 4x4 size array, and it's usually suitable for Vega
7816                          * ASICs which has 4*2 SE/SH layout.
7817                          * But for Arcturus, SE/SH layout is changed to 8*1.
7818                          * To mostly reduce the impact, we make it compatible
7819                          * with current bitmap array as below:
7820                          *    SE4,SH0 --> bitmap[0][1]
7821                          *    SE5,SH0 --> bitmap[1][1]
7822                          *    SE6,SH0 --> bitmap[2][1]
7823                          *    SE7,SH0 --> bitmap[3][1]
7824                          */
7825                         cu_info->bitmap[0][i % 4][j + i / 4] = bitmap;
7826
7827                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7828                                 if (bitmap & mask) {
7829                                         if (counter < adev->gfx.config.max_cu_per_sh)
7830                                                 ao_bitmap |= mask;
7831                                         counter ++;
7832                                 }
7833                                 mask <<= 1;
7834                         }
7835                         active_cu_number += counter;
7836                         if (i < 2 && j < 2)
7837                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7838                         cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7839                 }
7840         }
7841         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7842         mutex_unlock(&adev->grbm_idx_mutex);
7843
7844         cu_info->number = active_cu_number;
7845         cu_info->ao_cu_mask = ao_cu_mask;
7846         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7847
7848         return 0;
7849 }
7850
7851 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7852 {
7853         .type = AMD_IP_BLOCK_TYPE_GFX,
7854         .major = 9,
7855         .minor = 0,
7856         .rev = 0,
7857         .funcs = &gfx_v9_0_ip_funcs,
7858 };