drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "soc15.h"
  33 #include "soc15d.h"
  34 #include "amdgpu_atomfirmware.h"
  35 #include "amdgpu_pm.h"
  36
  37 #include "gc/gc_9_0_offset.h"
  38 #include "gc/gc_9_0_sh_mask.h"
  39
  40 #include "vega10_enum.h"
  41
  42 #include "soc15_common.h"
  43 #include "clearstate_gfx9.h"
  44 #include "v9_structs.h"
  45
  46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
  47
  48 #include "amdgpu_ras.h"
  49
  50 #include "amdgpu_ring_mux.h"
  51 #include "gfx_v9_4.h"
  52 #include "gfx_v9_0.h"
  53 #include "gfx_v9_0_cleaner_shader.h"
  54 #include "gfx_v9_4_2.h"
  55
  56 #include "asic_reg/pwr/pwr_10_0_offset.h"
  57 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
  58 #include "asic_reg/gc/gc_9_0_default.h"
  59
  60 #define GFX9_NUM_GFX_RINGS     1
  61 #define GFX9_NUM_SW_GFX_RINGS  2
  62 #define GFX9_MEC_HPD_SIZE 4096
  63 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
  64 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
  65
  66 #define mmGCEA_PROBE_MAP                        0x070c
  67 #define mmGCEA_PROBE_MAP_BASE_IDX               0
  68
  69 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  70 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  71 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  72 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  73 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  74 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  75
  76 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
  77 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
  78 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
  79 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
  80 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
  81 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
  82
  83 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
  84 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
  85 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
  86 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
  87 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
  88 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
  89
  90 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
  91 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
  92 MODULE_FIRMWARE("amdgpu/raven_me.bin");
  93 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
  94 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
  95 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
  96
  97 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
  98 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
  99 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
 100 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
 101 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
 102 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
 103 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
 104
 105 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
 106 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
 107 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
 108 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
 109 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
 110 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
 111 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
 112
 113 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
 114 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
 115
 116 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
 117 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
 118 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
 119 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
 120 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
 121
 122 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
 123 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
 124 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
 125 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
 126 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
 127 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
 128
 129 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
 130 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
 131 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
 132 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");
 133 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
 134
 135 #define mmTCP_CHAN_STEER_0_ARCT                                                         0x0b03
 136 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX                                                        0
 137 #define mmTCP_CHAN_STEER_1_ARCT                                                         0x0b04
 138 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX                                                        0
 139 #define mmTCP_CHAN_STEER_2_ARCT                                                         0x0b09
 140 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX                                                        0
 141 #define mmTCP_CHAN_STEER_3_ARCT                                                         0x0b0a
 142 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX                                                        0
 143 #define mmTCP_CHAN_STEER_4_ARCT                                                         0x0b0b
 144 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX                                                        0
 145 #define mmTCP_CHAN_STEER_5_ARCT                                                         0x0b0c
 146 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX                                                        0
 147
 148 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir                0x0025
 149 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX       1
 150 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir                0x0026
 151 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX       1
 152
 153 static const struct amdgpu_hwip_reg_entry gc_reg_list_9[] = {
 154         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS),
 155         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2),
 156         SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT1),
 157         SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT2),
 158         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STALLED_STAT1),
 159         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STALLED_STAT1),
 160         SOC15_REG_ENTRY_STR(GC, 0, mmCP_BUSY_STAT),
 161         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT),
 162         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT),
 163         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STATUS),
 164         SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_ERROR),
 165         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_BASE),
 166         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_RPTR),
 167         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_WPTR),
 168         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_BASE),
 169         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_RPTR),
 170         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_WPTR),
 171         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_BASE),
 172         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_RPTR),
 173         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_WPTR),
 174         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_BASE),
 175         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
 176         SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
 177         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_CMD_BUFSZ),
 178         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_CMD_BUFSZ),
 179         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_CMD_BUFSZ),
 180         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_CMD_BUFSZ),
 181         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_LO),
 182         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_HI),
 183         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BUFSZ),
 184         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_LO),
 185         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_HI),
 186         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BUFSZ),
 187         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_LO),
 188         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_HI),
 189         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BUFSZ),
 190         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_LO),
 191         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_HI),
 192         SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BUFSZ),
 193         SOC15_REG_ENTRY_STR(GC, 0, mmCPF_UTCL1_STATUS),
 194         SOC15_REG_ENTRY_STR(GC, 0, mmCPC_UTCL1_STATUS),
 195         SOC15_REG_ENTRY_STR(GC, 0, mmCPG_UTCL1_STATUS),
 196         SOC15_REG_ENTRY_STR(GC, 0, mmGDS_PROTECTION_FAULT),
 197         SOC15_REG_ENTRY_STR(GC, 0, mmGDS_VM_PROTECTION_FAULT),
 198         SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS),
 199         SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_CNTL),
 200         SOC15_REG_ENTRY_STR(GC, 0, mmPA_CL_CNTL_STATUS),
 201         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_UTCL1_STATUS),
 202         SOC15_REG_ENTRY_STR(GC, 0, mmRMI_UTCL1_STATUS),
 203         SOC15_REG_ENTRY_STR(GC, 0, mmSQC_DCACHE_UTCL1_STATUS),
 204         SOC15_REG_ENTRY_STR(GC, 0, mmSQC_ICACHE_UTCL1_STATUS),
 205         SOC15_REG_ENTRY_STR(GC, 0, mmSQ_UTCL1_STATUS),
 206         SOC15_REG_ENTRY_STR(GC, 0, mmTCP_UTCL1_STATUS),
 207         SOC15_REG_ENTRY_STR(GC, 0, mmWD_UTCL1_STATUS),
 208         SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL),
 209         SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS),
 210         SOC15_REG_ENTRY_STR(GC, 0, mmCP_DEBUG),
 211         SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_CNTL),
 212         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_INSTR_PNTR),
 213         SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC1_INSTR_PNTR),
 214         SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC2_INSTR_PNTR),
 215         SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_INSTR_PNTR),
 216         SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_INSTR_PNTR),
 217         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STATUS),
 218         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_STAT),
 219         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_COMMAND),
 220         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_MESSAGE),
 221         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_1),
 222         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_2),
 223         SOC15_REG_ENTRY_STR(GC, 0, mmSMU_RLC_RESPONSE),
 224         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SAFE_MODE),
 225         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_SAFE_MODE),
 226         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_INT_STAT),
 227         SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_GENERAL_6),
 228         /* cp header registers */
 229         SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
 230         SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
 231         SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME2_HEADER_DUMP),
 232         SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
 233         SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
 234         /* SE status registers */
 235         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE0),
 236         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE1),
 237         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE2),
 238         SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE3)
 239 };
 240
 241 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9[] = {
 242         /* compute queue registers */
 243         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_VMID),
 244         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ACTIVE),
 245         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PERSISTENT_STATE),
 246         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PIPE_PRIORITY),
 247         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUEUE_PRIORITY),
 248         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUANTUM),
 249         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE),
 250         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE_HI),
 251         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_RPTR),
 252         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
 253         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
 254         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL),
 255         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_CONTROL),
 256         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR),
 257         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR_HI),
 258         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_RPTR),
 259         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_CONTROL),
 260         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_DEQUEUE_REQUEST),
 261         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR),
 262         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI),
 263         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_CONTROL),
 264         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_RPTR),
 265         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR),
 266         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_EVENTS),
 267         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_LO),
 268         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_HI),
 269         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_CONTROL),
 270         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_OFFSET),
 271         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_SIZE),
 272         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_WG_STATE_OFFSET),
 273         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_SIZE),
 274         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GDS_RESOURCE_STATE),
 275         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ERROR),
 276         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR_MEM),
 277         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_LO),
 278         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_HI),
 279         SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GFX_STATUS),
 280 };
 281
 282 enum ta_ras_gfx_subblock {
 283         /*CPC*/
 284         TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
 285         TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
 286         TA_RAS_BLOCK__GFX_CPC_UCODE,
 287         TA_RAS_BLOCK__GFX_DC_STATE_ME1,
 288         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
 289         TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
 290         TA_RAS_BLOCK__GFX_DC_STATE_ME2,
 291         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
 292         TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 293         TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 294         /* CPF*/
 295         TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 296         TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 297         TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
 298         TA_RAS_BLOCK__GFX_CPF_TAG,
 299         TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
 300         /* CPG*/
 301         TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 302         TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 303         TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
 304         TA_RAS_BLOCK__GFX_CPG_TAG,
 305         TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
 306         /* GDS*/
 307         TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 308         TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 309         TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
 310         TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
 311         TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
 312         TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 313         TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 314         /* SPI*/
 315         TA_RAS_BLOCK__GFX_SPI_SR_MEM,
 316         /* SQ*/
 317         TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 318         TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 319         TA_RAS_BLOCK__GFX_SQ_LDS_D,
 320         TA_RAS_BLOCK__GFX_SQ_LDS_I,
 321         TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
 322         TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
 323         /* SQC (3 ranges)*/
 324         TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 325         /* SQC range 0*/
 326         TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 327         TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
 328                 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
 329         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
 330         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
 331         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
 332         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
 333         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
 334         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 335         TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
 336                 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 337         /* SQC range 1*/
 338         TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 339         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
 340                 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 341         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
 342         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
 343         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
 344         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
 345         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
 346         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
 347         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
 348         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 349         TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
 350                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 351         /* SQC range 2*/
 352         TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 353         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
 354                 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 355         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
 356         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
 357         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
 358         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
 359         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
 360         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
 361         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
 362         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 363         TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
 364                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 365         TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
 366         /* TA*/
 367         TA_RAS_BLOCK__GFX_TA_INDEX_START,
 368         TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
 369         TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
 370         TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
 371         TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
 372         TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 373         TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 374         /* TCA*/
 375         TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 376         TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 377         TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 378         TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 379         /* TCC (5 sub-ranges)*/
 380         TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 381         /* TCC range 0*/
 382         TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 383         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
 384         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
 385         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
 386         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
 387         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
 388         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
 389         TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
 390         TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 391         TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 392         /* TCC range 1*/
 393         TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 394         TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 395         TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 396         TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
 397                 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 398         /* TCC range 2*/
 399         TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 400         TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 401         TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
 402         TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
 403         TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
 404         TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
 405         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
 406         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
 407         TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 408         TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
 409                 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 410         /* TCC range 3*/
 411         TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 412         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 413         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 414         TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
 415                 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 416         /* TCC range 4*/
 417         TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 418         TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
 419                 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 420         TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 421         TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
 422                 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 423         TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
 424         /* TCI*/
 425         TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
 426         /* TCP*/
 427         TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 428         TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 429         TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
 430         TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
 431         TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
 432         TA_RAS_BLOCK__GFX_TCP_DB_RAM,
 433         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
 434         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 435         TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 436         /* TD*/
 437         TA_RAS_BLOCK__GFX_TD_INDEX_START,
 438         TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
 439         TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
 440         TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 441         TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 442         /* EA (3 sub-ranges)*/
 443         TA_RAS_BLOCK__GFX_EA_INDEX_START,
 444         /* EA range 0*/
 445         TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
 446         TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
 447         TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
 448         TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
 449         TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
 450         TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
 451         TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
 452         TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
 453         TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 454         TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 455         /* EA range 1*/
 456         TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 457         TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 458         TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
 459         TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
 460         TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
 461         TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
 462         TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
 463         TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 464         TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 465         /* EA range 2*/
 466         TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 467         TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 468         TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
 469         TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
 470         TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 471         TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 472         TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
 473         /* UTC VM L2 bank*/
 474         TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
 475         /* UTC VM walker*/
 476         TA_RAS_BLOCK__UTC_VML2_WALKER,
 477         /* UTC ATC L2 2MB cache*/
 478         TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
 479         /* UTC ATC L2 4KB cache*/
 480         TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
 481         TA_RAS_BLOCK__GFX_MAX
 482 };
 483
 484 struct ras_gfx_subblock {
 485         unsigned char *name;
 486         int ta_subblock;
 487         int hw_supported_error_type;
 488         int sw_supported_error_type;
 489 };
 490
 491 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \
 492         [AMDGPU_RAS_BLOCK__##subblock] = {                                     \
 493                 #subblock,                                                     \
 494                 TA_RAS_BLOCK__##subblock,                                      \
 495                 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \
 496                 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \
 497         }
 498
 499 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
 500         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
 501         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
 502         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 503         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 504         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 505         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 506         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 507         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 508         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 509         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 510         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
 511         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
 512         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
 513         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
 514         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 515         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
 516         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
 517                              0),
 518         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
 519                              0),
 520         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 521         AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 522         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 523         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
 524         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
 525         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 526         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
 527         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 528                              0, 0),
 529         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 530                              0),
 531         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 532                              0, 0),
 533         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
 534                              0),
 535         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 536                              0, 0),
 537         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 538                              0),
 539         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 540                              1),
 541         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 542                              0, 0, 0),
 543         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 544                              0),
 545         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 546                              0),
 547         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 548                              0),
 549         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 550                              0),
 551         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 552                              0),
 553         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 554                              0, 0),
 555         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 556                              0),
 557         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 558                              0),
 559         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 560                              0, 0, 0),
 561         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 562                              0),
 563         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 564                              0),
 565         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 566                              0),
 567         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 568                              0),
 569         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 570                              0),
 571         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 572                              0, 0),
 573         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 574                              0),
 575         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
 576         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 577         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 578         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 579         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 580         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
 581         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 582         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
 583         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
 584                              1),
 585         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
 586                              1),
 587         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
 588                              1),
 589         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
 590                              0),
 591         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
 592                              0),
 593         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 594         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 595         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
 596         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
 597         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
 598         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
 599         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 600         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
 601         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
 602         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 603         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
 604         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
 605                              0),
 606         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 607         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
 608                              0),
 609         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
 610                              0, 0),
 611         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
 612                              0),
 613         AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 614         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
 615         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
 616         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 617         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 618         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 619         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
 620         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
 621         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
 622         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
 623         AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 624         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
 625         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 626         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 627         AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 628         AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 629         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 630         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 631         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 632         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 633         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 634         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 635         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 636         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 637         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 638         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 639         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 640         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 641         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 642         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 643         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
 644         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
 645         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
 646         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
 647 };
 648
 649 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
 650 {
 651         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 652         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
 653         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 654         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 655         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 656         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 657         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 658         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 659         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 660         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
 661         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
 662         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 663         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 664         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 665         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 666         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 667         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
 668         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 669         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 670         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 671 };
 672
 673 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 674 {
 675         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 676         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 677         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 678         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 679         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 680         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 681         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
 682         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 683         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
 684         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 685         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 686         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 687         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 688         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 689         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 690         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
 691         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
 692         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 693 };
 694
 695 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
 696 {
 697         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
 698         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 699         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 700         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
 701         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
 702         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
 703         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
 704         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
 705         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
 706         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
 707         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
 708 };
 709
 710 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 711 {
 712         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 713         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 714         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 715         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 716         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 717         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 718         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 719         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 720         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 721         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 722         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 723         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 724         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 725         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 726         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 727         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 728         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 729         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 730         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 731         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
 732         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
 733         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 734         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 735         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 736 };
 737
 738 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 739 {
 740         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 741         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
 742         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
 743         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
 744         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
 745         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 746         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 747 };
 748
 749 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
 750 {
 751         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
 752         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 753         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 754         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
 755         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
 756         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
 757         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
 758         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
 759         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 760         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 761         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
 762         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
 763         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
 764         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 765         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 766         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 767         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
 768         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 769         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 770 };
 771
 772 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
 773 {
 774         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 775         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 776         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 777         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
 778         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
 779         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 780         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 781         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 782         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 783         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 784         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 785         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
 786 };
 787
 788 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 789 {
 790         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
 791         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
 792         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 793 };
 794
 795 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
 796 {
 797         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 798         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 799         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 800         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 801         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 802         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 803         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 804         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 805         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
 806         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 807         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 808         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 809         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 810         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 811         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 812         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 813 };
 814
 815 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 816 {
 817         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
 818         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 819         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 820         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
 821         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
 822         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 823         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
 824         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 825         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
 826         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 827         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 828         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 829         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 830 };
 831
 832 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
 833 {
 834         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 835         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
 836         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
 837         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
 838         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
 839         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
 840         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
 841         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
 842         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
 843         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
 844         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
 845 };
 846
 847 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
 848         {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
 849         {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
 850 };
 851
 852 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
 853 {
 854         mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 855         mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 856         mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 857         mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 858         mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 859         mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 860         mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 861         mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 862 };
 863
 864 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 865 {
 866         mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 867         mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 868         mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 869         mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 870         mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 871         mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 872         mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 873         mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 874 };
 875
 876 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 877 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 878 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 879 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
 880
 881 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 882 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 883 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 884 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 885 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 886                                 struct amdgpu_cu_info *cu_info);
 887 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 888 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);
 889 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
 890 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
 891                                           void *ras_error_status);
 892 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
 893                                      void *inject_if, uint32_t instance_mask);
 894 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
 895 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
 896                                               unsigned int vmid);
 897 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
 898 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
 899
 900 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 901                                 uint64_t queue_mask)
 902 {
 903         struct amdgpu_device *adev = kiq_ring->adev;
 904         u64 shader_mc_addr;
 905
 906         /* Cleaner shader MC address */
 907         shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
 908
 909         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 910         amdgpu_ring_write(kiq_ring,
 911                 PACKET3_SET_RESOURCES_VMID_MASK(0) |
 912                 /* vmid_mask:0* queue_type:0 (KIQ) */
 913                 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
 914         amdgpu_ring_write(kiq_ring,
 915                         lower_32_bits(queue_mask));     /* queue mask lo */
 916         amdgpu_ring_write(kiq_ring,
 917                         upper_32_bits(queue_mask));     /* queue mask hi */
 918         amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
 919         amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
 920         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 921         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 922 }
 923
 924 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 925                                  struct amdgpu_ring *ring)
 926 {
 927         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 928         uint64_t wptr_addr = ring->wptr_gpu_addr;
 929         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 930
 931         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 932         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 933         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 934                          PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 935                          PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 936                          PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 937                          PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 938                          PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
 939                          /*queue_type: normal compute queue */
 940                          PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
 941                          /* alloc format: all_on_one_pipe */
 942                          PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
 943                          PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 944                          /* num_queues: must be 1 */
 945                          PACKET3_MAP_QUEUES_NUM_QUEUES(1));
 946         amdgpu_ring_write(kiq_ring,
 947                         PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 948         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 949         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 950         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 951         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 952 }
 953
 954 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 955                                    struct amdgpu_ring *ring,
 956                                    enum amdgpu_unmap_queues_action action,
 957                                    u64 gpu_addr, u64 seq)
 958 {
 959         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 960
 961         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 962         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 963                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 964                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 965                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 966                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 967         amdgpu_ring_write(kiq_ring,
 968                         PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 969
 970         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 971                 amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));
 972                 amdgpu_ring_write(kiq_ring, 0);
 973                 amdgpu_ring_write(kiq_ring, 0);
 974
 975         } else {
 976                 amdgpu_ring_write(kiq_ring, 0);
 977                 amdgpu_ring_write(kiq_ring, 0);
 978                 amdgpu_ring_write(kiq_ring, 0);
 979         }
 980 }
 981
 982 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 983                                    struct amdgpu_ring *ring,
 984                                    u64 addr,
 985                                    u64 seq)
 986 {
 987         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 988
 989         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 990         amdgpu_ring_write(kiq_ring,
 991                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 992                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 993                           PACKET3_QUERY_STATUS_COMMAND(2));
 994         /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 995         amdgpu_ring_write(kiq_ring,
 996                         PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 997                         PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 998         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 999         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
1000         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
1001         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
1002 }
1003
1004 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
1005                                 uint16_t pasid, uint32_t flush_type,
1006                                 bool all_hub)
1007 {
1008         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
1009         amdgpu_ring_write(kiq_ring,
1010                         PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
1011                         PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
1012                         PACKET3_INVALIDATE_TLBS_PASID(pasid) |
1013                         PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
1014 }
1015
1016
1017 static void gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type,
1018                                         uint32_t me_id, uint32_t pipe_id, uint32_t queue_id,
1019                                         uint32_t xcc_id, uint32_t vmid)
1020 {
1021         struct amdgpu_device *adev = kiq_ring->adev;
1022         unsigned i;
1023
1024         /* enter save mode */
1025         amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
1026         mutex_lock(&adev->srbm_mutex);
1027         soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, 0);
1028
1029         if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
1030                 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0x2);
1031                 WREG32_SOC15(GC, 0, mmSPI_COMPUTE_QUEUE_RESET, 0x1);
1032                 /* wait till dequeue take effects */
1033                 for (i = 0; i < adev->usec_timeout; i++) {
1034                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
1035                                 break;
1036                         udelay(1);
1037                 }
1038                 if (i >= adev->usec_timeout)
1039                         dev_err(adev->dev, "fail to wait on hqd deactive\n");
1040         } else {
1041                 dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type);
1042         }
1043
1044         soc15_grbm_select(adev, 0, 0, 0, 0, 0);
1045         mutex_unlock(&adev->srbm_mutex);
1046         /* exit safe mode */
1047         amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
1048 }
1049
1050 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
1051         .kiq_set_resources = gfx_v9_0_kiq_set_resources,
1052         .kiq_map_queues = gfx_v9_0_kiq_map_queues,
1053         .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
1054         .kiq_query_status = gfx_v9_0_kiq_query_status,
1055         .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
1056         .kiq_reset_hw_queue = gfx_v9_0_kiq_reset_hw_queue,
1057         .set_resources_size = 8,
1058         .map_queues_size = 7,
1059         .unmap_queues_size = 6,
1060         .query_status_size = 7,
1061         .invalidate_tlbs_size = 2,
1062 };
1063
1064 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
1065 {
1066         adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs;
1067 }
1068
1069 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
1070 {
1071         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1072         case IP_VERSION(9, 0, 1):
1073                 soc15_program_register_sequence(adev,
1074                                                 golden_settings_gc_9_0,
1075                                                 ARRAY_SIZE(golden_settings_gc_9_0));
1076                 soc15_program_register_sequence(adev,
1077                                                 golden_settings_gc_9_0_vg10,
1078                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
1079                 break;
1080         case IP_VERSION(9, 2, 1):
1081                 soc15_program_register_sequence(adev,
1082                                                 golden_settings_gc_9_2_1,
1083                                                 ARRAY_SIZE(golden_settings_gc_9_2_1));
1084                 soc15_program_register_sequence(adev,
1085                                                 golden_settings_gc_9_2_1_vg12,
1086                                                 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
1087                 break;
1088         case IP_VERSION(9, 4, 0):
1089                 soc15_program_register_sequence(adev,
1090                                                 golden_settings_gc_9_0,
1091                                                 ARRAY_SIZE(golden_settings_gc_9_0));
1092                 soc15_program_register_sequence(adev,
1093                                                 golden_settings_gc_9_0_vg20,
1094                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
1095                 break;
1096         case IP_VERSION(9, 4, 1):
1097                 soc15_program_register_sequence(adev,
1098                                                 golden_settings_gc_9_4_1_arct,
1099                                                 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
1100                 break;
1101         case IP_VERSION(9, 2, 2):
1102         case IP_VERSION(9, 1, 0):
1103                 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
1104                                                 ARRAY_SIZE(golden_settings_gc_9_1));
1105                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1106                         soc15_program_register_sequence(adev,
1107                                                         golden_settings_gc_9_1_rv2,
1108                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv2));
1109                 else
1110                         soc15_program_register_sequence(adev,
1111                                                         golden_settings_gc_9_1_rv1,
1112                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv1));
1113                 break;
1114          case IP_VERSION(9, 3, 0):
1115                 soc15_program_register_sequence(adev,
1116                                                 golden_settings_gc_9_1_rn,
1117                                                 ARRAY_SIZE(golden_settings_gc_9_1_rn));
1118                 return; /* for renoir, don't need common goldensetting */
1119         case IP_VERSION(9, 4, 2):
1120                 gfx_v9_4_2_init_golden_registers(adev,
1121                                                  adev->smuio.funcs->get_die_id(adev));
1122                 break;
1123         default:
1124                 break;
1125         }
1126
1127         if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1128             (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)))
1129                 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
1130                                                 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
1131 }
1132
1133 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1134                                        bool wc, uint32_t reg, uint32_t val)
1135 {
1136         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1137         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1138                                 WRITE_DATA_DST_SEL(0) |
1139                                 (wc ? WR_CONFIRM : 0));
1140         amdgpu_ring_write(ring, reg);
1141         amdgpu_ring_write(ring, 0);
1142         amdgpu_ring_write(ring, val);
1143 }
1144
1145 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1146                                   int mem_space, int opt, uint32_t addr0,
1147                                   uint32_t addr1, uint32_t ref, uint32_t mask,
1148                                   uint32_t inv)
1149 {
1150         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1151         amdgpu_ring_write(ring,
1152                                  /* memory (1) or register (0) */
1153                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1154                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
1155                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
1156                                  WAIT_REG_MEM_ENGINE(eng_sel)));
1157
1158         if (mem_space)
1159                 BUG_ON(addr0 & 0x3); /* Dword align */
1160         amdgpu_ring_write(ring, addr0);
1161         amdgpu_ring_write(ring, addr1);
1162         amdgpu_ring_write(ring, ref);
1163         amdgpu_ring_write(ring, mask);
1164         amdgpu_ring_write(ring, inv); /* poll interval */
1165 }
1166
1167 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1168 {
1169         struct amdgpu_device *adev = ring->adev;
1170         uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1171         uint32_t tmp = 0;
1172         unsigned i;
1173         int r;
1174
1175         WREG32(scratch, 0xCAFEDEAD);
1176         r = amdgpu_ring_alloc(ring, 3);
1177         if (r)
1178                 return r;
1179
1180         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1181         amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START);
1182         amdgpu_ring_write(ring, 0xDEADBEEF);
1183         amdgpu_ring_commit(ring);
1184
1185         for (i = 0; i < adev->usec_timeout; i++) {
1186                 tmp = RREG32(scratch);
1187                 if (tmp == 0xDEADBEEF)
1188                         break;
1189                 udelay(1);
1190         }
1191
1192         if (i >= adev->usec_timeout)
1193                 r = -ETIMEDOUT;
1194         return r;
1195 }
1196
1197 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1198 {
1199         struct amdgpu_device *adev = ring->adev;
1200         struct amdgpu_ib ib;
1201         struct dma_fence *f = NULL;
1202
1203         unsigned index;
1204         uint64_t gpu_addr;
1205         uint32_t tmp;
1206         long r;
1207
1208         r = amdgpu_device_wb_get(adev, &index);
1209         if (r)
1210                 return r;
1211
1212         gpu_addr = adev->wb.gpu_addr + (index * 4);
1213         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1214         memset(&ib, 0, sizeof(ib));
1215
1216         r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
1217         if (r)
1218                 goto err1;
1219
1220         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1221         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1222         ib.ptr[2] = lower_32_bits(gpu_addr);
1223         ib.ptr[3] = upper_32_bits(gpu_addr);
1224         ib.ptr[4] = 0xDEADBEEF;
1225         ib.length_dw = 5;
1226
1227         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1228         if (r)
1229                 goto err2;
1230
1231         r = dma_fence_wait_timeout(f, false, timeout);
1232         if (r == 0) {
1233                 r = -ETIMEDOUT;
1234                 goto err2;
1235         } else if (r < 0) {
1236                 goto err2;
1237         }
1238
1239         tmp = adev->wb.wb[index];
1240         if (tmp == 0xDEADBEEF)
1241                 r = 0;
1242         else
1243                 r = -EINVAL;
1244
1245 err2:
1246         amdgpu_ib_free(&ib, NULL);
1247         dma_fence_put(f);
1248 err1:
1249         amdgpu_device_wb_free(adev, index);
1250         return r;
1251 }
1252
1253
1254 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1255 {
1256         amdgpu_ucode_release(&adev->gfx.pfp_fw);
1257         amdgpu_ucode_release(&adev->gfx.me_fw);
1258         amdgpu_ucode_release(&adev->gfx.ce_fw);
1259         amdgpu_ucode_release(&adev->gfx.rlc_fw);
1260         amdgpu_ucode_release(&adev->gfx.mec_fw);
1261         amdgpu_ucode_release(&adev->gfx.mec2_fw);
1262
1263         kfree(adev->gfx.rlc.register_list_format);
1264 }
1265
1266 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1267 {
1268         adev->gfx.me_fw_write_wait = false;
1269         adev->gfx.mec_fw_write_wait = false;
1270
1271         if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1272             ((adev->gfx.mec_fw_version < 0x000001a5) ||
1273              (adev->gfx.mec_feature_version < 46) ||
1274              (adev->gfx.pfp_fw_version < 0x000000b7) ||
1275              (adev->gfx.pfp_feature_version < 46)))
1276                 DRM_WARN_ONCE("CP firmware version too old, please update!");
1277
1278         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1279         case IP_VERSION(9, 0, 1):
1280                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1281                     (adev->gfx.me_feature_version >= 42) &&
1282                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1283                     (adev->gfx.pfp_feature_version >= 42))
1284                         adev->gfx.me_fw_write_wait = true;
1285
1286                 if ((adev->gfx.mec_fw_version >=  0x00000193) &&
1287                     (adev->gfx.mec_feature_version >= 42))
1288                         adev->gfx.mec_fw_write_wait = true;
1289                 break;
1290         case IP_VERSION(9, 2, 1):
1291                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1292                     (adev->gfx.me_feature_version >= 44) &&
1293                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1294                     (adev->gfx.pfp_feature_version >= 44))
1295                         adev->gfx.me_fw_write_wait = true;
1296
1297                 if ((adev->gfx.mec_fw_version >=  0x00000196) &&
1298                     (adev->gfx.mec_feature_version >= 44))
1299                         adev->gfx.mec_fw_write_wait = true;
1300                 break;
1301         case IP_VERSION(9, 4, 0):
1302                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1303                     (adev->gfx.me_feature_version >= 44) &&
1304                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1305                     (adev->gfx.pfp_feature_version >= 44))
1306                         adev->gfx.me_fw_write_wait = true;
1307
1308                 if ((adev->gfx.mec_fw_version >=  0x00000197) &&
1309                     (adev->gfx.mec_feature_version >= 44))
1310                         adev->gfx.mec_fw_write_wait = true;
1311                 break;
1312         case IP_VERSION(9, 1, 0):
1313         case IP_VERSION(9, 2, 2):
1314                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1315                     (adev->gfx.me_feature_version >= 42) &&
1316                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1317                     (adev->gfx.pfp_feature_version >= 42))
1318                         adev->gfx.me_fw_write_wait = true;
1319
1320                 if ((adev->gfx.mec_fw_version >=  0x00000192) &&
1321                     (adev->gfx.mec_feature_version >= 42))
1322                         adev->gfx.mec_fw_write_wait = true;
1323                 break;
1324         default:
1325                 adev->gfx.me_fw_write_wait = true;
1326                 adev->gfx.mec_fw_write_wait = true;
1327                 break;
1328         }
1329 }
1330
1331 struct amdgpu_gfxoff_quirk {
1332         u16 chip_vendor;
1333         u16 chip_device;
1334         u16 subsys_vendor;
1335         u16 subsys_device;
1336         u8 revision;
1337 };
1338
1339 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1340         /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1341         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1342         /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1343         { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1344         /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1345         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1346         /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1347         { 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
1348         /* https://bbs.openkylin.top/t/topic/171497 */
1349         { 0x1002, 0x15d8, 0x19e5, 0x3e14, 0xc2 },
1350         /* HP 705G4 DM with R5 2400G */
1351         { 0x1002, 0x15dd, 0x103c, 0x8464, 0xd6 },
1352         { 0, 0, 0, 0, 0 },
1353 };
1354
1355 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1356 {
1357         const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1358
1359         while (p && p->chip_device != 0) {
1360                 if (pdev->vendor == p->chip_vendor &&
1361                     pdev->device == p->chip_device &&
1362                     pdev->subsystem_vendor == p->subsys_vendor &&
1363                     pdev->subsystem_device == p->subsys_device &&
1364                     pdev->revision == p->revision) {
1365                         return true;
1366                 }
1367                 ++p;
1368         }
1369         return false;
1370 }
1371
1372 static bool is_raven_kicker(struct amdgpu_device *adev)
1373 {
1374         if (adev->pm.fw_version >= 0x41e2b)
1375                 return true;
1376         else
1377                 return false;
1378 }
1379
1380 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1381 {
1382         if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) &&
1383             (adev->gfx.me_fw_version >= 0x000000a5) &&
1384             (adev->gfx.me_feature_version >= 52))
1385                 return true;
1386         else
1387                 return false;
1388 }
1389
1390 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1391 {
1392         if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1393                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1394
1395         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1396         case IP_VERSION(9, 0, 1):
1397         case IP_VERSION(9, 2, 1):
1398         case IP_VERSION(9, 4, 0):
1399                 break;
1400         case IP_VERSION(9, 2, 2):
1401         case IP_VERSION(9, 1, 0):
1402                 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1403                       (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1404                     ((!is_raven_kicker(adev) &&
1405                       adev->gfx.rlc_fw_version < 531) ||
1406                      (adev->gfx.rlc_feature_version < 1) ||
1407                      !adev->gfx.rlc.is_rlc_v2_1))
1408                         adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1409
1410                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1411                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1412                                 AMD_PG_SUPPORT_CP |
1413                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1414                 break;
1415         case IP_VERSION(9, 3, 0):
1416                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1417                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1418                                 AMD_PG_SUPPORT_CP |
1419                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1420                 break;
1421         default:
1422                 break;
1423         }
1424 }
1425
1426 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1427                                           char *chip_name)
1428 {
1429         int err;
1430
1431         err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
1432                                    AMDGPU_UCODE_REQUIRED,
1433                                    "amdgpu/%s_pfp.bin", chip_name);
1434         if (err)
1435                 goto out;
1436         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
1437
1438         err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
1439                                    AMDGPU_UCODE_REQUIRED,
1440                                    "amdgpu/%s_me.bin", chip_name);
1441         if (err)
1442                 goto out;
1443         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
1444
1445         err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,
1446                                    AMDGPU_UCODE_REQUIRED,
1447                                    "amdgpu/%s_ce.bin", chip_name);
1448         if (err)
1449                 goto out;
1450         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE);
1451
1452 out:
1453         if (err) {
1454                 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1455                 amdgpu_ucode_release(&adev->gfx.me_fw);
1456                 amdgpu_ucode_release(&adev->gfx.ce_fw);
1457         }
1458         return err;
1459 }
1460
1461 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1462                                        char *chip_name)
1463 {
1464         int err;
1465         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1466         uint16_t version_major;
1467         uint16_t version_minor;
1468         uint32_t smu_version;
1469
1470         /*
1471          * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1472          * instead of picasso_rlc.bin.
1473          * Judgment method:
1474          * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1475          *          or revision >= 0xD8 && revision <= 0xDF
1476          * otherwise is PCO FP5
1477          */
1478         if (!strcmp(chip_name, "picasso") &&
1479                 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1480                 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1481                 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1482                                            AMDGPU_UCODE_REQUIRED,
1483                                            "amdgpu/%s_rlc_am4.bin", chip_name);
1484         else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1485                 (smu_version >= 0x41e2b))
1486                 /**
1487                 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1488                 */
1489                 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1490                                            AMDGPU_UCODE_REQUIRED,
1491                                            "amdgpu/%s_kicker_rlc.bin", chip_name);
1492         else
1493                 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1494                                            AMDGPU_UCODE_REQUIRED,
1495                                            "amdgpu/%s_rlc.bin", chip_name);
1496         if (err)
1497                 goto out;
1498
1499         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1500         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1501         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1502         err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
1503 out:
1504         if (err)
1505                 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1506
1507         return err;
1508 }
1509
1510 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1511 {
1512         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
1513             amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
1514             amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0))
1515                 return false;
1516
1517         return true;
1518 }
1519
1520 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1521                                               char *chip_name)
1522 {
1523         int err;
1524
1525         if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1526                 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1527                                    AMDGPU_UCODE_REQUIRED,
1528                                    "amdgpu/%s_sjt_mec.bin", chip_name);
1529         else
1530                 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1531                                            AMDGPU_UCODE_REQUIRED,
1532                                            "amdgpu/%s_mec.bin", chip_name);
1533         if (err)
1534                 goto out;
1535
1536         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
1537         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
1538
1539         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1540                 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1541                         err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1542                                                    AMDGPU_UCODE_REQUIRED,
1543                                                    "amdgpu/%s_sjt_mec2.bin", chip_name);
1544                 else
1545                         err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1546                                                    AMDGPU_UCODE_REQUIRED,
1547                                                    "amdgpu/%s_mec2.bin", chip_name);
1548                 if (!err) {
1549                         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2);
1550                         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT);
1551                 } else {
1552                         err = 0;
1553                         amdgpu_ucode_release(&adev->gfx.mec2_fw);
1554                 }
1555         } else {
1556                 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1557                 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1558         }
1559
1560         gfx_v9_0_check_if_need_gfxoff(adev);
1561         gfx_v9_0_check_fw_write_wait(adev);
1562
1563 out:
1564         if (err)
1565                 amdgpu_ucode_release(&adev->gfx.mec_fw);
1566         return err;
1567 }
1568
1569 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1570 {
1571         char ucode_prefix[30];
1572         int r;
1573
1574         DRM_DEBUG("\n");
1575         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
1576
1577         /* No CPG in Arcturus */
1578         if (adev->gfx.num_gfx_rings) {
1579                 r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix);
1580                 if (r)
1581                         return r;
1582         }
1583
1584         r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix);
1585         if (r)
1586                 return r;
1587
1588         r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix);
1589         if (r)
1590                 return r;
1591
1592         return r;
1593 }
1594
1595 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1596 {
1597         u32 count = 0;
1598         const struct cs_section_def *sect = NULL;
1599         const struct cs_extent_def *ext = NULL;
1600
1601         /* begin clear state */
1602         count += 2;
1603         /* context control state */
1604         count += 3;
1605
1606         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1607                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1608                         if (sect->id == SECT_CONTEXT)
1609                                 count += 2 + ext->reg_count;
1610                         else
1611                                 return 0;
1612                 }
1613         }
1614
1615         /* end clear state */
1616         count += 2;
1617         /* clear state */
1618         count += 2;
1619
1620         return count;
1621 }
1622
1623 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1624                                     volatile u32 *buffer)
1625 {
1626         u32 count = 0, i;
1627         const struct cs_section_def *sect = NULL;
1628         const struct cs_extent_def *ext = NULL;
1629
1630         if (adev->gfx.rlc.cs_data == NULL)
1631                 return;
1632         if (buffer == NULL)
1633                 return;
1634
1635         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1636         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1637
1638         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1639         buffer[count++] = cpu_to_le32(0x80000000);
1640         buffer[count++] = cpu_to_le32(0x80000000);
1641
1642         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1643                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1644                         if (sect->id == SECT_CONTEXT) {
1645                                 buffer[count++] =
1646                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1647                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1648                                                 PACKET3_SET_CONTEXT_REG_START);
1649                                 for (i = 0; i < ext->reg_count; i++)
1650                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1651                         } else {
1652                                 return;
1653                         }
1654                 }
1655         }
1656
1657         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1658         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1659
1660         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1661         buffer[count++] = cpu_to_le32(0);
1662 }
1663
1664 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1665 {
1666         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1667         uint32_t pg_always_on_cu_num = 2;
1668         uint32_t always_on_cu_num;
1669         uint32_t i, j, k;
1670         uint32_t mask, cu_bitmap, counter;
1671
1672         if (adev->flags & AMD_IS_APU)
1673                 always_on_cu_num = 4;
1674         else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1))
1675                 always_on_cu_num = 8;
1676         else
1677                 always_on_cu_num = 12;
1678
1679         mutex_lock(&adev->grbm_idx_mutex);
1680         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1681                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1682                         mask = 1;
1683                         cu_bitmap = 0;
1684                         counter = 0;
1685                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
1686
1687                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1688                                 if (cu_info->bitmap[0][i][j] & mask) {
1689                                         if (counter == pg_always_on_cu_num)
1690                                                 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1691                                         if (counter < always_on_cu_num)
1692                                                 cu_bitmap |= mask;
1693                                         else
1694                                                 break;
1695                                         counter++;
1696                                 }
1697                                 mask <<= 1;
1698                         }
1699
1700                         WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1701                         cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1702                 }
1703         }
1704         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1705         mutex_unlock(&adev->grbm_idx_mutex);
1706 }
1707
1708 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1709 {
1710         uint32_t data;
1711
1712         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1713         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1714         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1715         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1716         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1717
1718         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1719         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1720
1721         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1722         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1723
1724         mutex_lock(&adev->grbm_idx_mutex);
1725         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1726         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1727         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1728
1729         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1730         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1731         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1732         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1733         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1734
1735         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1736         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1737         data &= 0x0000FFFF;
1738         data |= 0x00C00000;
1739         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1740
1741         /*
1742          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1743          * programmed in gfx_v9_0_init_always_on_cu_mask()
1744          */
1745
1746         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1747          * but used for RLC_LB_CNTL configuration */
1748         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1749         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1750         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1751         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1752         mutex_unlock(&adev->grbm_idx_mutex);
1753
1754         gfx_v9_0_init_always_on_cu_mask(adev);
1755 }
1756
1757 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1758 {
1759         uint32_t data;
1760
1761         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1762         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1763         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1764         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1765         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1766
1767         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1768         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1769
1770         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1771         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1772
1773         mutex_lock(&adev->grbm_idx_mutex);
1774         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1775         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1776         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1777
1778         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1779         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1780         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1781         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1782         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1783
1784         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1785         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1786         data &= 0x0000FFFF;
1787         data |= 0x00C00000;
1788         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1789
1790         /*
1791          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1792          * programmed in gfx_v9_0_init_always_on_cu_mask()
1793          */
1794
1795         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1796          * but used for RLC_LB_CNTL configuration */
1797         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1798         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1799         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1800         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1801         mutex_unlock(&adev->grbm_idx_mutex);
1802
1803         gfx_v9_0_init_always_on_cu_mask(adev);
1804 }
1805
1806 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1807 {
1808         WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1809 }
1810
1811 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1812 {
1813         if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1814                 return 5;
1815         else
1816                 return 4;
1817 }
1818
1819 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1820 {
1821         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1822
1823         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
1824         reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1825         reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);
1826         reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);
1827         reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);
1828         reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);
1829         reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);
1830         reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);
1831         adev->gfx.rlc.rlcg_reg_access_supported = true;
1832 }
1833
1834 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1835 {
1836         const struct cs_section_def *cs_data;
1837         int r;
1838
1839         adev->gfx.rlc.cs_data = gfx9_cs_data;
1840
1841         cs_data = adev->gfx.rlc.cs_data;
1842
1843         if (cs_data) {
1844                 /* init clear state block */
1845                 r = amdgpu_gfx_rlc_init_csb(adev);
1846                 if (r)
1847                         return r;
1848         }
1849
1850         if (adev->flags & AMD_IS_APU) {
1851                 /* TODO: double check the cp_table_size for RV */
1852                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1853                 r = amdgpu_gfx_rlc_init_cpt(adev);
1854                 if (r)
1855                         return r;
1856         }
1857
1858         return 0;
1859 }
1860
1861 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1862 {
1863         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1864         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1865 }
1866
1867 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1868 {
1869         int r;
1870         u32 *hpd;
1871         const __le32 *fw_data;
1872         unsigned fw_size;
1873         u32 *fw;
1874         size_t mec_hpd_size;
1875
1876         const struct gfx_firmware_header_v1_0 *mec_hdr;
1877
1878         bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1879
1880         /* take ownership of the relevant compute queues */
1881         amdgpu_gfx_compute_queue_acquire(adev);
1882         mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1883         if (mec_hpd_size) {
1884                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1885                                               AMDGPU_GEM_DOMAIN_VRAM |
1886                                               AMDGPU_GEM_DOMAIN_GTT,
1887                                               &adev->gfx.mec.hpd_eop_obj,
1888                                               &adev->gfx.mec.hpd_eop_gpu_addr,
1889                                               (void **)&hpd);
1890                 if (r) {
1891                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1892                         gfx_v9_0_mec_fini(adev);
1893                         return r;
1894                 }
1895
1896                 memset(hpd, 0, mec_hpd_size);
1897
1898                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1899                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1900         }
1901
1902         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1903
1904         fw_data = (const __le32 *)
1905                 (adev->gfx.mec_fw->data +
1906                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1907         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1908
1909         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1910                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1911                                       &adev->gfx.mec.mec_fw_obj,
1912                                       &adev->gfx.mec.mec_fw_gpu_addr,
1913                                       (void **)&fw);
1914         if (r) {
1915                 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1916                 gfx_v9_0_mec_fini(adev);
1917                 return r;
1918         }
1919
1920         memcpy(fw, fw_data, fw_size);
1921
1922         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1923         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1924
1925         return 0;
1926 }
1927
1928 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1929 {
1930         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1931                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1932                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1933                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
1934                 (SQ_IND_INDEX__FORCE_READ_MASK));
1935         return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1936 }
1937
1938 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1939                            uint32_t wave, uint32_t thread,
1940                            uint32_t regno, uint32_t num, uint32_t *out)
1941 {
1942         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1943                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1944                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1945                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1946                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1947                 (SQ_IND_INDEX__FORCE_READ_MASK) |
1948                 (SQ_IND_INDEX__AUTO_INCR_MASK));
1949         while (num--)
1950                 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1951 }
1952
1953 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1954 {
1955         /* type 1 wave data */
1956         dst[(*no_fields)++] = 1;
1957         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1958         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1959         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1960         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1961         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1962         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1963         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1964         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1965         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1966         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1967         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1968         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1969         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1970         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1971         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
1972 }
1973
1974 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1975                                      uint32_t wave, uint32_t start,
1976                                      uint32_t size, uint32_t *dst)
1977 {
1978         wave_read_regs(
1979                 adev, simd, wave, 0,
1980                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
1981 }
1982
1983 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1984                                      uint32_t wave, uint32_t thread,
1985                                      uint32_t start, uint32_t size,
1986                                      uint32_t *dst)
1987 {
1988         wave_read_regs(
1989                 adev, simd, wave, thread,
1990                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1991 }
1992
1993 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
1994                                   u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1995 {
1996         soc15_grbm_select(adev, me, pipe, q, vm, 0);
1997 }
1998
1999 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2000         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2001         .select_se_sh = &gfx_v9_0_select_se_sh,
2002         .read_wave_data = &gfx_v9_0_read_wave_data,
2003         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2004         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2005         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2006 };
2007
2008 const struct amdgpu_ras_block_hw_ops  gfx_v9_0_ras_ops = {
2009                 .ras_error_inject = &gfx_v9_0_ras_error_inject,
2010                 .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2011                 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2012 };
2013
2014 static struct amdgpu_gfx_ras gfx_v9_0_ras = {
2015         .ras_block = {
2016                 .hw_ops = &gfx_v9_0_ras_ops,
2017         },
2018 };
2019
2020 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2021 {
2022         u32 gb_addr_config;
2023         int err;
2024
2025         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2026         case IP_VERSION(9, 0, 1):
2027                 adev->gfx.config.max_hw_contexts = 8;
2028                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2029                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2030                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2031                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2032                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2033                 break;
2034         case IP_VERSION(9, 2, 1):
2035                 adev->gfx.config.max_hw_contexts = 8;
2036                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2037                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2038                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2039                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2040                 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2041                 DRM_INFO("fix gfx.config for vega12\n");
2042                 break;
2043         case IP_VERSION(9, 4, 0):
2044                 adev->gfx.ras = &gfx_v9_0_ras;
2045                 adev->gfx.config.max_hw_contexts = 8;
2046                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2047                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2048                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2049                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2050                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2051                 gb_addr_config &= ~0xf3e777ff;
2052                 gb_addr_config |= 0x22014042;
2053                 /* check vbios table if gpu info is not available */
2054                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2055                 if (err)
2056                         return err;
2057                 break;
2058         case IP_VERSION(9, 2, 2):
2059         case IP_VERSION(9, 1, 0):
2060                 adev->gfx.config.max_hw_contexts = 8;
2061                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2062                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2063                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2064                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2065                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2066                         gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2067                 else
2068                         gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2069                 break;
2070         case IP_VERSION(9, 4, 1):
2071                 adev->gfx.ras = &gfx_v9_4_ras;
2072                 adev->gfx.config.max_hw_contexts = 8;
2073                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2074                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2075                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2076                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2077                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2078                 gb_addr_config &= ~0xf3e777ff;
2079                 gb_addr_config |= 0x22014042;
2080                 break;
2081         case IP_VERSION(9, 3, 0):
2082                 adev->gfx.config.max_hw_contexts = 8;
2083                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2084                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2085                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2086                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2087                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2088                 gb_addr_config &= ~0xf3e777ff;
2089                 gb_addr_config |= 0x22010042;
2090                 break;
2091         case IP_VERSION(9, 4, 2):
2092                 adev->gfx.ras = &gfx_v9_4_2_ras;
2093                 adev->gfx.config.max_hw_contexts = 8;
2094                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2095                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2096                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2097                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2098                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2099                 gb_addr_config &= ~0xf3e777ff;
2100                 gb_addr_config |= 0x22014042;
2101                 /* check vbios table if gpu info is not available */
2102                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2103                 if (err)
2104                         return err;
2105                 break;
2106         default:
2107                 BUG();
2108                 break;
2109         }
2110
2111         adev->gfx.config.gb_addr_config = gb_addr_config;
2112
2113         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2114                         REG_GET_FIELD(
2115                                         adev->gfx.config.gb_addr_config,
2116                                         GB_ADDR_CONFIG,
2117                                         NUM_PIPES);
2118
2119         adev->gfx.config.max_tile_pipes =
2120                 adev->gfx.config.gb_addr_config_fields.num_pipes;
2121
2122         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2123                         REG_GET_FIELD(
2124                                         adev->gfx.config.gb_addr_config,
2125                                         GB_ADDR_CONFIG,
2126                                         NUM_BANKS);
2127         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2128                         REG_GET_FIELD(
2129                                         adev->gfx.config.gb_addr_config,
2130                                         GB_ADDR_CONFIG,
2131                                         MAX_COMPRESSED_FRAGS);
2132         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2133                         REG_GET_FIELD(
2134                                         adev->gfx.config.gb_addr_config,
2135                                         GB_ADDR_CONFIG,
2136                                         NUM_RB_PER_SE);
2137         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2138                         REG_GET_FIELD(
2139                                         adev->gfx.config.gb_addr_config,
2140                                         GB_ADDR_CONFIG,
2141                                         NUM_SHADER_ENGINES);
2142         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2143                         REG_GET_FIELD(
2144                                         adev->gfx.config.gb_addr_config,
2145                                         GB_ADDR_CONFIG,
2146                                         PIPE_INTERLEAVE_SIZE));
2147
2148         return 0;
2149 }
2150
2151 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2152                                       int mec, int pipe, int queue)
2153 {
2154         unsigned irq_type;
2155         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2156         unsigned int hw_prio;
2157
2158         ring = &adev->gfx.compute_ring[ring_id];
2159
2160         /* mec0 is me1 */
2161         ring->me = mec + 1;
2162         ring->pipe = pipe;
2163         ring->queue = queue;
2164
2165         ring->ring_obj = NULL;
2166         ring->use_doorbell = true;
2167         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2168         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2169                                 + (ring_id * GFX9_MEC_HPD_SIZE);
2170         ring->vm_hub = AMDGPU_GFXHUB(0);
2171         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2172
2173         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2174                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2175                 + ring->pipe;
2176         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2177                         AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
2178         /* type-2 packets are deprecated on MEC, use type-3 instead */
2179         return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2180                                 hw_prio, NULL);
2181 }
2182
2183 static void gfx_v9_0_alloc_ip_dump(struct amdgpu_device *adev)
2184 {
2185         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
2186         uint32_t *ptr;
2187         uint32_t inst;
2188
2189         ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
2190         if (!ptr) {
2191                 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
2192                 adev->gfx.ip_dump_core = NULL;
2193         } else {
2194                 adev->gfx.ip_dump_core = ptr;
2195         }
2196
2197         /* Allocate memory for compute queue registers for all the instances */
2198         reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
2199         inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
2200                 adev->gfx.mec.num_queue_per_pipe;
2201
2202         ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
2203         if (!ptr) {
2204                 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
2205                 adev->gfx.ip_dump_compute_queues = NULL;
2206         } else {
2207                 adev->gfx.ip_dump_compute_queues = ptr;
2208         }
2209 }
2210
2211 static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
2212 {
2213         int i, j, k, r, ring_id;
2214         int xcc_id = 0;
2215         struct amdgpu_ring *ring;
2216         struct amdgpu_device *adev = ip_block->adev;
2217         unsigned int hw_prio;
2218
2219         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2220         case IP_VERSION(9, 0, 1):
2221         case IP_VERSION(9, 2, 1):
2222         case IP_VERSION(9, 4, 0):
2223         case IP_VERSION(9, 2, 2):
2224         case IP_VERSION(9, 1, 0):
2225         case IP_VERSION(9, 4, 1):
2226         case IP_VERSION(9, 3, 0):
2227         case IP_VERSION(9, 4, 2):
2228                 adev->gfx.mec.num_mec = 2;
2229                 break;
2230         default:
2231                 adev->gfx.mec.num_mec = 1;
2232                 break;
2233         }
2234
2235         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2236         case IP_VERSION(9, 4, 2):
2237                 adev->gfx.cleaner_shader_ptr = gfx_9_4_2_cleaner_shader_hex;
2238                 adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_2_cleaner_shader_hex);
2239                 if (adev->gfx.mec_fw_version >= 88) {
2240                         adev->gfx.enable_cleaner_shader = true;
2241                         r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
2242                         if (r) {
2243                                 adev->gfx.enable_cleaner_shader = false;
2244                                 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
2245                         }
2246                 }
2247                 break;
2248         default:
2249                 adev->gfx.enable_cleaner_shader = false;
2250                 break;
2251         }
2252
2253         adev->gfx.mec.num_pipe_per_mec = 4;
2254         adev->gfx.mec.num_queue_per_pipe = 8;
2255
2256         /* EOP Event */
2257         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2258         if (r)
2259                 return r;
2260
2261         /* Bad opcode Event */
2262         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
2263                               GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR,
2264                               &adev->gfx.bad_op_irq);
2265         if (r)
2266                 return r;
2267
2268         /* Privileged reg */
2269         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2270                               &adev->gfx.priv_reg_irq);
2271         if (r)
2272                 return r;
2273
2274         /* Privileged inst */
2275         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2276                               &adev->gfx.priv_inst_irq);
2277         if (r)
2278                 return r;
2279
2280         /* ECC error */
2281         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2282                               &adev->gfx.cp_ecc_error_irq);
2283         if (r)
2284                 return r;
2285
2286         /* FUE error */
2287         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2288                               &adev->gfx.cp_ecc_error_irq);
2289         if (r)
2290                 return r;
2291
2292         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2293
2294         if (adev->gfx.rlc.funcs) {
2295                 if (adev->gfx.rlc.funcs->init) {
2296                         r = adev->gfx.rlc.funcs->init(adev);
2297                         if (r) {
2298                                 dev_err(adev->dev, "Failed to init rlc BOs!\n");
2299                                 return r;
2300                         }
2301                 }
2302         }
2303
2304         r = gfx_v9_0_mec_init(adev);
2305         if (r) {
2306                 DRM_ERROR("Failed to init MEC BOs!\n");
2307                 return r;
2308         }
2309
2310         /* set up the gfx ring */
2311         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2312                 ring = &adev->gfx.gfx_ring[i];
2313                 ring->ring_obj = NULL;
2314                 if (!i)
2315                         sprintf(ring->name, "gfx");
2316                 else
2317                         sprintf(ring->name, "gfx_%d", i);
2318                 ring->use_doorbell = true;
2319                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2320
2321                 /* disable scheduler on the real ring */
2322                 ring->no_scheduler = adev->gfx.mcbp;
2323                 ring->vm_hub = AMDGPU_GFXHUB(0);
2324                 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2325                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2326                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
2327                 if (r)
2328                         return r;
2329         }
2330
2331         /* set up the software rings */
2332         if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2333                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2334                         ring = &adev->gfx.sw_gfx_ring[i];
2335                         ring->ring_obj = NULL;
2336                         sprintf(ring->name, amdgpu_sw_ring_name(i));
2337                         ring->use_doorbell = true;
2338                         ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2339                         ring->is_sw_ring = true;
2340                         hw_prio = amdgpu_sw_ring_priority(i);
2341                         ring->vm_hub = AMDGPU_GFXHUB(0);
2342                         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2343                                              AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,
2344                                              NULL);
2345                         if (r)
2346                                 return r;
2347                         ring->wptr = 0;
2348                 }
2349
2350                 /* init the muxer and add software rings */
2351                 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0],
2352                                          GFX9_NUM_SW_GFX_RINGS);
2353                 if (r) {
2354                         DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r);
2355                         return r;
2356                 }
2357                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2358                         r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer,
2359                                                         &adev->gfx.sw_gfx_ring[i]);
2360                         if (r) {
2361                                 DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r);
2362                                 return r;
2363                         }
2364                 }
2365         }
2366
2367         /* set up the compute queues - allocate horizontally across pipes */
2368         ring_id = 0;
2369         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2370                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2371                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2372                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
2373                                                                      k, j))
2374                                         continue;
2375
2376                                 r = gfx_v9_0_compute_ring_init(adev,
2377                                                                ring_id,
2378                                                                i, k, j);
2379                                 if (r)
2380                                         return r;
2381
2382                                 ring_id++;
2383                         }
2384                 }
2385         }
2386
2387         /* TODO: Add queue reset mask when FW fully supports it */
2388         adev->gfx.gfx_supported_reset =
2389                 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]);
2390         adev->gfx.compute_supported_reset =
2391                 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
2392
2393         r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0);
2394         if (r) {
2395                 DRM_ERROR("Failed to init KIQ BOs!\n");
2396                 return r;
2397         }
2398
2399         r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
2400         if (r)
2401                 return r;
2402
2403         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2404         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation), 0);
2405         if (r)
2406                 return r;
2407
2408         adev->gfx.ce_ram_size = 0x8000;
2409
2410         r = gfx_v9_0_gpu_early_init(adev);
2411         if (r)
2412                 return r;
2413
2414         if (amdgpu_gfx_ras_sw_init(adev)) {
2415                 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
2416                 return -EINVAL;
2417         }
2418
2419         gfx_v9_0_alloc_ip_dump(adev);
2420
2421         r = amdgpu_gfx_sysfs_init(adev);
2422         if (r)
2423                 return r;
2424
2425         return 0;
2426 }
2427
2428
2429 static int gfx_v9_0_sw_fini(struct amdgpu_ip_block *ip_block)
2430 {
2431         int i;
2432         struct amdgpu_device *adev = ip_block->adev;
2433
2434         if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2435                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
2436                         amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]);
2437                 amdgpu_ring_mux_fini(&adev->gfx.muxer);
2438         }
2439
2440         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2441                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2442         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2443                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2444
2445         amdgpu_gfx_mqd_sw_fini(adev, 0);
2446         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
2447         amdgpu_gfx_kiq_fini(adev, 0);
2448
2449         amdgpu_gfx_cleaner_shader_sw_fini(adev);
2450
2451         gfx_v9_0_mec_fini(adev);
2452         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2453                                 &adev->gfx.rlc.clear_state_gpu_addr,
2454                                 (void **)&adev->gfx.rlc.cs_ptr);
2455         if (adev->flags & AMD_IS_APU) {
2456                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2457                                 &adev->gfx.rlc.cp_table_gpu_addr,
2458                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2459         }
2460         gfx_v9_0_free_microcode(adev);
2461
2462         amdgpu_gfx_sysfs_fini(adev);
2463
2464         kfree(adev->gfx.ip_dump_core);
2465         kfree(adev->gfx.ip_dump_compute_queues);
2466
2467         return 0;
2468 }
2469
2470
2471 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2472 {
2473         /* TODO */
2474 }
2475
2476 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2477                            u32 instance, int xcc_id)
2478 {
2479         u32 data;
2480
2481         if (instance == 0xffffffff)
2482                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2483         else
2484                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2485
2486         if (se_num == 0xffffffff)
2487                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2488         else
2489                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2490
2491         if (sh_num == 0xffffffff)
2492                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2493         else
2494                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2495
2496         WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2497 }
2498
2499 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2500 {
2501         u32 data, mask;
2502
2503         data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2504         data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2505
2506         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2507         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2508
2509         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2510                                          adev->gfx.config.max_sh_per_se);
2511
2512         return (~data) & mask;
2513 }
2514
2515 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2516 {
2517         int i, j;
2518         u32 data;
2519         u32 active_rbs = 0;
2520         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2521                                         adev->gfx.config.max_sh_per_se;
2522
2523         mutex_lock(&adev->grbm_idx_mutex);
2524         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2525                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2526                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2527                         data = gfx_v9_0_get_rb_active_bitmap(adev);
2528                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2529                                                rb_bitmap_width_per_sh);
2530                 }
2531         }
2532         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2533         mutex_unlock(&adev->grbm_idx_mutex);
2534
2535         adev->gfx.config.backend_enable_mask = active_rbs;
2536         adev->gfx.config.num_rbs = hweight32(active_rbs);
2537 }
2538
2539 static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
2540                                 uint32_t first_vmid,
2541                                 uint32_t last_vmid)
2542 {
2543         uint32_t data;
2544         uint32_t trap_config_vmid_mask = 0;
2545         int i;
2546
2547         /* Calculate trap config vmid mask */
2548         for (i = first_vmid; i < last_vmid; i++)
2549                 trap_config_vmid_mask |= (1 << i);
2550
2551         data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
2552                         VMID_SEL, trap_config_vmid_mask);
2553         data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
2554                         TRAP_EN, 1);
2555         WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
2556         WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
2557
2558         WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0);
2559         WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0);
2560 }
2561
2562 #define DEFAULT_SH_MEM_BASES    (0x6000)
2563 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2564 {
2565         int i;
2566         uint32_t sh_mem_config;
2567         uint32_t sh_mem_bases;
2568
2569         /*
2570          * Configure apertures:
2571          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
2572          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
2573          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
2574          */
2575         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2576
2577         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2578                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2579                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2580
2581         mutex_lock(&adev->srbm_mutex);
2582         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2583                 soc15_grbm_select(adev, 0, 0, 0, i, 0);
2584                 /* CP and shaders */
2585                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2586                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2587         }
2588         soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2589         mutex_unlock(&adev->srbm_mutex);
2590
2591         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2592            access. These should be enabled by FW for target VMIDs. */
2593         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2594                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2595                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2596                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2597                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2598         }
2599 }
2600
2601 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2602 {
2603         int vmid;
2604
2605         /*
2606          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2607          * access. Compute VMIDs should be enabled by FW for target VMIDs,
2608          * the driver can enable them for graphics. VMID0 should maintain
2609          * access so that HWS firmware can save/restore entries.
2610          */
2611         for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2612                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2613                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2614                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2615                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2616         }
2617 }
2618
2619 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2620 {
2621         uint32_t tmp;
2622
2623         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2624         case IP_VERSION(9, 4, 1):
2625                 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2626                 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT,
2627                                 !READ_ONCE(adev->barrier_has_auto_waitcnt));
2628                 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2629                 break;
2630         default:
2631                 break;
2632         }
2633 }
2634
2635 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2636 {
2637         u32 tmp;
2638         int i;
2639
2640         WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2641
2642         gfx_v9_0_tiling_mode_table_init(adev);
2643
2644         if (adev->gfx.num_gfx_rings)
2645                 gfx_v9_0_setup_rb(adev);
2646         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2647         adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2648
2649         /* XXX SH_MEM regs */
2650         /* where to put LDS, scratch, GPUVM in FSA64 space */
2651         mutex_lock(&adev->srbm_mutex);
2652         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
2653                 soc15_grbm_select(adev, 0, 0, 0, i, 0);
2654                 /* CP and shaders */
2655                 if (i == 0) {
2656                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2657                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2658                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2659                                             !!adev->gmc.noretry);
2660                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2661                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2662                 } else {
2663                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2664                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2665                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2666                                             !!adev->gmc.noretry);
2667                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2668                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2669                                 (adev->gmc.private_aperture_start >> 48));
2670                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2671                                 (adev->gmc.shared_aperture_start >> 48));
2672                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2673                 }
2674         }
2675         soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2676
2677         mutex_unlock(&adev->srbm_mutex);
2678
2679         gfx_v9_0_init_compute_vmid(adev);
2680         gfx_v9_0_init_gds_vmid(adev);
2681         gfx_v9_0_init_sq_config(adev);
2682 }
2683
2684 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2685 {
2686         u32 i, j, k;
2687         u32 mask;
2688
2689         mutex_lock(&adev->grbm_idx_mutex);
2690         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2691                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2692                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2693                         for (k = 0; k < adev->usec_timeout; k++) {
2694                                 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2695                                         break;
2696                                 udelay(1);
2697                         }
2698                         if (k == adev->usec_timeout) {
2699                                 amdgpu_gfx_select_se_sh(adev, 0xffffffff,
2700                                                       0xffffffff, 0xffffffff, 0);
2701                                 mutex_unlock(&adev->grbm_idx_mutex);
2702                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2703                                          i, j);
2704                                 return;
2705                         }
2706                 }
2707         }
2708         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2709         mutex_unlock(&adev->grbm_idx_mutex);
2710
2711         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2712                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2713                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2714                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2715         for (k = 0; k < adev->usec_timeout; k++) {
2716                 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2717                         break;
2718                 udelay(1);
2719         }
2720 }
2721
2722 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2723                                                bool enable)
2724 {
2725         u32 tmp;
2726
2727         /* These interrupts should be enabled to drive DS clock */
2728
2729         tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2730
2731         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2732         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2733         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2734         if (adev->gfx.num_gfx_rings)
2735                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2736
2737         WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2738 }
2739
2740 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2741 {
2742         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2743         /* csib */
2744         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2745                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
2746         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2747                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2748         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2749                         adev->gfx.rlc.clear_state_size);
2750 }
2751
2752 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2753                                 int indirect_offset,
2754                                 int list_size,
2755                                 int *unique_indirect_regs,
2756                                 int unique_indirect_reg_count,
2757                                 int *indirect_start_offsets,
2758                                 int *indirect_start_offsets_count,
2759                                 int max_start_offsets_count)
2760 {
2761         int idx;
2762
2763         for (; indirect_offset < list_size; indirect_offset++) {
2764                 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2765                 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2766                 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2767
2768                 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2769                         indirect_offset += 2;
2770
2771                         /* look for the matching indice */
2772                         for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2773                                 if (unique_indirect_regs[idx] ==
2774                                         register_list_format[indirect_offset] ||
2775                                         !unique_indirect_regs[idx])
2776                                         break;
2777                         }
2778
2779                         BUG_ON(idx >= unique_indirect_reg_count);
2780
2781                         if (!unique_indirect_regs[idx])
2782                                 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2783
2784                         indirect_offset++;
2785                 }
2786         }
2787 }
2788
2789 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2790 {
2791         int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2792         int unique_indirect_reg_count = 0;
2793
2794         int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2795         int indirect_start_offsets_count = 0;
2796
2797         int list_size = 0;
2798         int i = 0, j = 0;
2799         u32 tmp = 0;
2800
2801         u32 *register_list_format =
2802                 kmemdup(adev->gfx.rlc.register_list_format,
2803                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2804         if (!register_list_format)
2805                 return -ENOMEM;
2806
2807         /* setup unique_indirect_regs array and indirect_start_offsets array */
2808         unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2809         gfx_v9_1_parse_ind_reg_list(register_list_format,
2810                                     adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2811                                     adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2812                                     unique_indirect_regs,
2813                                     unique_indirect_reg_count,
2814                                     indirect_start_offsets,
2815                                     &indirect_start_offsets_count,
2816                                     ARRAY_SIZE(indirect_start_offsets));
2817
2818         /* enable auto inc in case it is disabled */
2819         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2820         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2821         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2822
2823         /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2824         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2825                 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2826         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2827                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2828                         adev->gfx.rlc.register_restore[i]);
2829
2830         /* load indirect register */
2831         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2832                 adev->gfx.rlc.reg_list_format_start);
2833
2834         /* direct register portion */
2835         for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2836                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2837                         register_list_format[i]);
2838
2839         /* indirect register portion */
2840         while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2841                 if (register_list_format[i] == 0xFFFFFFFF) {
2842                         WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2843                         continue;
2844                 }
2845
2846                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2847                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2848
2849                 for (j = 0; j < unique_indirect_reg_count; j++) {
2850                         if (register_list_format[i] == unique_indirect_regs[j]) {
2851                                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2852                                 break;
2853                         }
2854                 }
2855
2856                 BUG_ON(j >= unique_indirect_reg_count);
2857
2858                 i++;
2859         }
2860
2861         /* set save/restore list size */
2862         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2863         list_size = list_size >> 1;
2864         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2865                 adev->gfx.rlc.reg_restore_list_size);
2866         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2867
2868         /* write the starting offsets to RLC scratch ram */
2869         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2870                 adev->gfx.rlc.starting_offsets_start);
2871         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2872                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2873                        indirect_start_offsets[i]);
2874
2875         /* load unique indirect regs*/
2876         for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2877                 if (unique_indirect_regs[i] != 0) {
2878                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2879                                + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2880                                unique_indirect_regs[i] & 0x3FFFF);
2881
2882                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2883                                + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2884                                unique_indirect_regs[i] >> 20);
2885                 }
2886         }
2887
2888         kfree(register_list_format);
2889         return 0;
2890 }
2891
2892 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2893 {
2894         WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2895 }
2896
2897 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2898                                              bool enable)
2899 {
2900         uint32_t data = 0;
2901         uint32_t default_data = 0;
2902
2903         default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2904         if (enable) {
2905                 /* enable GFXIP control over CGPG */
2906                 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2907                 if(default_data != data)
2908                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2909
2910                 /* update status */
2911                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2912                 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2913                 if(default_data != data)
2914                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2915         } else {
2916                 /* restore GFXIP control over GCPG */
2917                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2918                 if(default_data != data)
2919                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2920         }
2921 }
2922
2923 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2924 {
2925         uint32_t data = 0;
2926
2927         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2928                               AMD_PG_SUPPORT_GFX_SMG |
2929                               AMD_PG_SUPPORT_GFX_DMG)) {
2930                 /* init IDLE_POLL_COUNT = 60 */
2931                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2932                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2933                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2934                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2935
2936                 /* init RLC PG Delay */
2937                 data = 0;
2938                 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2939                 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2940                 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2941                 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2942                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2943
2944                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2945                 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2946                 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2947                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2948
2949                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2950                 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2951                 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2952                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2953
2954                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2955                 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2956
2957                 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2958                 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2959                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2960                 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0))
2961                         pwr_10_0_gfxip_control_over_cgpg(adev, true);
2962         }
2963 }
2964
2965 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2966                                                 bool enable)
2967 {
2968         uint32_t data = 0;
2969         uint32_t default_data = 0;
2970
2971         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2972         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2973                              SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2974                              enable ? 1 : 0);
2975         if (default_data != data)
2976                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2977 }
2978
2979 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2980                                                 bool enable)
2981 {
2982         uint32_t data = 0;
2983         uint32_t default_data = 0;
2984
2985         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2986         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2987                              SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2988                              enable ? 1 : 0);
2989         if(default_data != data)
2990                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2991 }
2992
2993 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2994                                         bool enable)
2995 {
2996         uint32_t data = 0;
2997         uint32_t default_data = 0;
2998
2999         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3000         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3001                              CP_PG_DISABLE,
3002                              enable ? 0 : 1);
3003         if(default_data != data)
3004                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3005 }
3006
3007 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
3008                                                 bool enable)
3009 {
3010         uint32_t data, default_data;
3011
3012         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3013         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3014                              GFX_POWER_GATING_ENABLE,
3015                              enable ? 1 : 0);
3016         if(default_data != data)
3017                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3018 }
3019
3020 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
3021                                                 bool enable)
3022 {
3023         uint32_t data, default_data;
3024
3025         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3026         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3027                              GFX_PIPELINE_PG_ENABLE,
3028                              enable ? 1 : 0);
3029         if(default_data != data)
3030                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3031
3032         if (!enable)
3033                 /* read any GFX register to wake up GFX */
3034                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
3035 }
3036
3037 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3038                                                        bool enable)
3039 {
3040         uint32_t data, default_data;
3041
3042         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3043         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3044                              STATIC_PER_CU_PG_ENABLE,
3045                              enable ? 1 : 0);
3046         if(default_data != data)
3047                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3048 }
3049
3050 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3051                                                 bool enable)
3052 {
3053         uint32_t data, default_data;
3054
3055         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3056         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3057                              DYN_PER_CU_PG_ENABLE,
3058                              enable ? 1 : 0);
3059         if(default_data != data)
3060                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3061 }
3062
3063 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3064 {
3065         gfx_v9_0_init_csb(adev);
3066
3067         /*
3068          * Rlc save restore list is workable since v2_1.
3069          * And it's needed by gfxoff feature.
3070          */
3071         if (adev->gfx.rlc.is_rlc_v2_1) {
3072                 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
3073                             IP_VERSION(9, 2, 1) ||
3074                     (adev->apu_flags & AMD_APU_IS_RAVEN2))
3075                         gfx_v9_1_init_rlc_save_restore_list(adev);
3076                 gfx_v9_0_enable_save_restore_machine(adev);
3077         }
3078
3079         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3080                               AMD_PG_SUPPORT_GFX_SMG |
3081                               AMD_PG_SUPPORT_GFX_DMG |
3082                               AMD_PG_SUPPORT_CP |
3083                               AMD_PG_SUPPORT_GDS |
3084                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
3085                 WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,
3086                              adev->gfx.rlc.cp_table_gpu_addr >> 8);
3087                 gfx_v9_0_init_gfx_power_gating(adev);
3088         }
3089 }
3090
3091 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3092 {
3093         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3094         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3095         gfx_v9_0_wait_for_rlc_serdes(adev);
3096 }
3097
3098 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3099 {
3100         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3101         udelay(50);
3102         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3103         udelay(50);
3104 }
3105
3106 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3107 {
3108 #ifdef AMDGPU_RLC_DEBUG_RETRY
3109         u32 rlc_ucode_ver;
3110 #endif
3111
3112         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3113         udelay(50);
3114
3115         /* carrizo do enable cp interrupt after cp inited */
3116         if (!(adev->flags & AMD_IS_APU)) {
3117                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3118                 udelay(50);
3119         }
3120
3121 #ifdef AMDGPU_RLC_DEBUG_RETRY
3122         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3123         rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3124         if(rlc_ucode_ver == 0x108) {
3125                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3126                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3127                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3128                  * default is 0x9C4 to create a 100us interval */
3129                 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3130                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3131                  * to disable the page fault retry interrupts, default is
3132                  * 0x100 (256) */
3133                 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3134         }
3135 #endif
3136 }
3137
3138 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3139 {
3140         const struct rlc_firmware_header_v2_0 *hdr;
3141         const __le32 *fw_data;
3142         unsigned i, fw_size;
3143
3144         if (!adev->gfx.rlc_fw)
3145                 return -EINVAL;
3146
3147         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3148         amdgpu_ucode_print_rlc_hdr(&hdr->header);
3149
3150         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3151                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3152         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3153
3154         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3155                         RLCG_UCODE_LOADING_START_ADDRESS);
3156         for (i = 0; i < fw_size; i++)
3157                 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3158         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3159
3160         return 0;
3161 }
3162
3163 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3164 {
3165         int r;
3166
3167         if (amdgpu_sriov_vf(adev)) {
3168                 gfx_v9_0_init_csb(adev);
3169                 return 0;
3170         }
3171
3172         adev->gfx.rlc.funcs->stop(adev);
3173
3174         /* disable CG */
3175         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3176
3177         gfx_v9_0_init_pg(adev);
3178
3179         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3180                 /* legacy rlc firmware loading */
3181                 r = gfx_v9_0_rlc_load_microcode(adev);
3182                 if (r)
3183                         return r;
3184         }
3185
3186         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3187         case IP_VERSION(9, 2, 2):
3188         case IP_VERSION(9, 1, 0):
3189                 gfx_v9_0_init_lbpw(adev);
3190                 if (amdgpu_lbpw == 0)
3191                         gfx_v9_0_enable_lbpw(adev, false);
3192                 else
3193                         gfx_v9_0_enable_lbpw(adev, true);
3194                 break;
3195         case IP_VERSION(9, 4, 0):
3196                 gfx_v9_4_init_lbpw(adev);
3197                 if (amdgpu_lbpw > 0)
3198                         gfx_v9_0_enable_lbpw(adev, true);
3199                 else
3200                         gfx_v9_0_enable_lbpw(adev, false);
3201                 break;
3202         default:
3203                 break;
3204         }
3205
3206         gfx_v9_0_update_spm_vmid_internal(adev, 0xf);
3207
3208         adev->gfx.rlc.funcs->start(adev);
3209
3210         return 0;
3211 }
3212
3213 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3214 {
3215         u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3216
3217         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_INVALIDATE_ICACHE, enable ? 0 : 1);
3218         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_INVALIDATE_ICACHE, enable ? 0 : 1);
3219         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_INVALIDATE_ICACHE, enable ? 0 : 1);
3220         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE0_RESET, enable ? 0 : 1);
3221         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE1_RESET, enable ? 0 : 1);
3222         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, enable ? 0 : 1);
3223         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, enable ? 0 : 1);
3224         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, enable ? 0 : 1);
3225         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, enable ? 0 : 1);
3226         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3227         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3228         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3229         WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3230         udelay(50);
3231 }
3232
3233 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3234 {
3235         const struct gfx_firmware_header_v1_0 *pfp_hdr;
3236         const struct gfx_firmware_header_v1_0 *ce_hdr;
3237         const struct gfx_firmware_header_v1_0 *me_hdr;
3238         const __le32 *fw_data;
3239         unsigned i, fw_size;
3240
3241         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3242                 return -EINVAL;
3243
3244         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3245                 adev->gfx.pfp_fw->data;
3246         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3247                 adev->gfx.ce_fw->data;
3248         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3249                 adev->gfx.me_fw->data;
3250
3251         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3252         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3253         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3254
3255         gfx_v9_0_cp_gfx_enable(adev, false);
3256
3257         /* PFP */
3258         fw_data = (const __le32 *)
3259                 (adev->gfx.pfp_fw->data +
3260                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3261         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3262         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3263         for (i = 0; i < fw_size; i++)
3264                 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3265         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3266
3267         /* CE */
3268         fw_data = (const __le32 *)
3269                 (adev->gfx.ce_fw->data +
3270                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3271         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3272         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3273         for (i = 0; i < fw_size; i++)
3274                 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3275         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3276
3277         /* ME */
3278         fw_data = (const __le32 *)
3279                 (adev->gfx.me_fw->data +
3280                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3281         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3282         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3283         for (i = 0; i < fw_size; i++)
3284                 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3285         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3286
3287         return 0;
3288 }
3289
3290 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3291 {
3292         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3293         const struct cs_section_def *sect = NULL;
3294         const struct cs_extent_def *ext = NULL;
3295         int r, i, tmp;
3296
3297         /* init the CP */
3298         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3299         WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3300
3301         gfx_v9_0_cp_gfx_enable(adev, true);
3302
3303         /* Now only limit the quirk on the APU gfx9 series and already
3304          * confirmed that the APU gfx10/gfx11 needn't such update.
3305          */
3306         if (adev->flags & AMD_IS_APU &&
3307                         adev->in_s3 && !pm_resume_via_firmware()) {
3308                 DRM_INFO("Will skip the CSB packet resubmit\n");
3309                 return 0;
3310         }
3311         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3312         if (r) {
3313                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3314                 return r;
3315         }
3316
3317         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3318         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3319
3320         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3321         amdgpu_ring_write(ring, 0x80000000);
3322         amdgpu_ring_write(ring, 0x80000000);
3323
3324         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3325                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3326                         if (sect->id == SECT_CONTEXT) {
3327                                 amdgpu_ring_write(ring,
3328                                        PACKET3(PACKET3_SET_CONTEXT_REG,
3329                                                ext->reg_count));
3330                                 amdgpu_ring_write(ring,
3331                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3332                                 for (i = 0; i < ext->reg_count; i++)
3333                                         amdgpu_ring_write(ring, ext->extent[i]);
3334                         }
3335                 }
3336         }
3337
3338         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3339         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3340
3341         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3342         amdgpu_ring_write(ring, 0);
3343
3344         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3345         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3346         amdgpu_ring_write(ring, 0x8000);
3347         amdgpu_ring_write(ring, 0x8000);
3348
3349         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3350         tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3351                 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3352         amdgpu_ring_write(ring, tmp);
3353         amdgpu_ring_write(ring, 0);
3354
3355         amdgpu_ring_commit(ring);
3356
3357         return 0;
3358 }
3359
3360 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3361 {
3362         struct amdgpu_ring *ring;
3363         u32 tmp;
3364         u32 rb_bufsz;
3365         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3366
3367         /* Set the write pointer delay */
3368         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3369
3370         /* set the RB to use vmid 0 */
3371         WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3372
3373         /* Set ring buffer size */
3374         ring = &adev->gfx.gfx_ring[0];
3375         rb_bufsz = order_base_2(ring->ring_size / 8);
3376         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3377         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3378 #ifdef __BIG_ENDIAN
3379         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3380 #endif
3381         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3382
3383         /* Initialize the ring buffer's write pointers */
3384         ring->wptr = 0;
3385         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3386         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3387
3388         /* set the wb address whether it's enabled or not */
3389         rptr_addr = ring->rptr_gpu_addr;
3390         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3391         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3392
3393         wptr_gpu_addr = ring->wptr_gpu_addr;
3394         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3395         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3396
3397         mdelay(1);
3398         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3399
3400         rb_addr = ring->gpu_addr >> 8;
3401         WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3402         WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3403
3404         tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3405         if (ring->use_doorbell) {
3406                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3407                                     DOORBELL_OFFSET, ring->doorbell_index);
3408                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3409                                     DOORBELL_EN, 1);
3410         } else {
3411                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3412         }
3413         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3414
3415         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3416                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
3417         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3418
3419         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3420                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3421
3422
3423         /* start the ring */
3424         gfx_v9_0_cp_gfx_start(adev);
3425
3426         return 0;
3427 }
3428
3429 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3430 {
3431         if (enable) {
3432                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3433         } else {
3434                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3435                                  (CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK |
3436                                   CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK |
3437                                   CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK |
3438                                   CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK |
3439                                   CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK |
3440                                   CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK |
3441                                   CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK |
3442                                   CP_MEC_CNTL__MEC_ME1_HALT_MASK |
3443                                   CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3444                 adev->gfx.kiq[0].ring.sched.ready = false;
3445         }
3446         udelay(50);
3447 }
3448
3449 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3450 {
3451         const struct gfx_firmware_header_v1_0 *mec_hdr;
3452         const __le32 *fw_data;
3453         unsigned i;
3454         u32 tmp;
3455
3456         if (!adev->gfx.mec_fw)
3457                 return -EINVAL;
3458
3459         gfx_v9_0_cp_compute_enable(adev, false);
3460
3461         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3462         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3463
3464         fw_data = (const __le32 *)
3465                 (adev->gfx.mec_fw->data +
3466                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3467         tmp = 0;
3468         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3469         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3470         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3471
3472         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3473                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3474         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3475                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3476
3477         /* MEC1 */
3478         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3479                          mec_hdr->jt_offset);
3480         for (i = 0; i < mec_hdr->jt_size; i++)
3481                 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3482                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3483
3484         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3485                         adev->gfx.mec_fw_version);
3486         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3487
3488         return 0;
3489 }
3490
3491 /* KIQ functions */
3492 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3493 {
3494         uint32_t tmp;
3495         struct amdgpu_device *adev = ring->adev;
3496
3497         /* tell RLC which is KIQ queue */
3498         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3499         tmp &= 0xffffff00;
3500         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3501         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp | 0x80);
3502 }
3503
3504 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3505 {
3506         struct amdgpu_device *adev = ring->adev;
3507
3508         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3509                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3510                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3511                         mqd->cp_hqd_queue_priority =
3512                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3513                 }
3514         }
3515 }
3516
3517 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3518 {
3519         struct amdgpu_device *adev = ring->adev;
3520         struct v9_mqd *mqd = ring->mqd_ptr;
3521         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3522         uint32_t tmp;
3523
3524         mqd->header = 0xC0310800;
3525         mqd->compute_pipelinestat_enable = 0x00000001;
3526         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3527         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3528         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3529         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3530         mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3531         mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3532         mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3533         mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3534         mqd->compute_misc_reserved = 0x00000003;
3535
3536         mqd->dynamic_cu_mask_addr_lo =
3537                 lower_32_bits(ring->mqd_gpu_addr
3538                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3539         mqd->dynamic_cu_mask_addr_hi =
3540                 upper_32_bits(ring->mqd_gpu_addr
3541                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3542
3543         eop_base_addr = ring->eop_gpu_addr >> 8;
3544         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3545         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3546
3547         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3548         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3549         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3550                         (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3551
3552         mqd->cp_hqd_eop_control = tmp;
3553
3554         /* enable doorbell? */
3555         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3556
3557         if (ring->use_doorbell) {
3558                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3559                                     DOORBELL_OFFSET, ring->doorbell_index);
3560                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3561                                     DOORBELL_EN, 1);
3562                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3563                                     DOORBELL_SOURCE, 0);
3564                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3565                                     DOORBELL_HIT, 0);
3566         } else {
3567                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3568                                          DOORBELL_EN, 0);
3569         }
3570
3571         mqd->cp_hqd_pq_doorbell_control = tmp;
3572
3573         /* disable the queue if it's active */
3574         ring->wptr = 0;
3575         mqd->cp_hqd_dequeue_request = 0;
3576         mqd->cp_hqd_pq_rptr = 0;
3577         mqd->cp_hqd_pq_wptr_lo = 0;
3578         mqd->cp_hqd_pq_wptr_hi = 0;
3579
3580         /* set the pointer to the MQD */
3581         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3582         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3583
3584         /* set MQD vmid to 0 */
3585         tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3586         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3587         mqd->cp_mqd_control = tmp;
3588
3589         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3590         hqd_gpu_addr = ring->gpu_addr >> 8;
3591         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3592         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3593
3594         /* set up the HQD, this is similar to CP_RB0_CNTL */
3595         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3596         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3597                             (order_base_2(ring->ring_size / 4) - 1));
3598         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3599                         (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3600 #ifdef __BIG_ENDIAN
3601         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3602 #endif
3603         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3604         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3605         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3606         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3607         mqd->cp_hqd_pq_control = tmp;
3608
3609         /* set the wb address whether it's enabled or not */
3610         wb_gpu_addr = ring->rptr_gpu_addr;
3611         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3612         mqd->cp_hqd_pq_rptr_report_addr_hi =
3613                 upper_32_bits(wb_gpu_addr) & 0xffff;
3614
3615         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3616         wb_gpu_addr = ring->wptr_gpu_addr;
3617         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3618         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3619
3620         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3621         ring->wptr = 0;
3622         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3623
3624         /* set the vmid for the queue */
3625         mqd->cp_hqd_vmid = 0;
3626
3627         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3628         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3629         mqd->cp_hqd_persistent_state = tmp;
3630
3631         /* set MIN_IB_AVAIL_SIZE */
3632         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3633         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3634         mqd->cp_hqd_ib_control = tmp;
3635
3636         /* set static priority for a queue/ring */
3637         gfx_v9_0_mqd_set_priority(ring, mqd);
3638         mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
3639
3640         /* map_queues packet doesn't need activate the queue,
3641          * so only kiq need set this field.
3642          */
3643         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3644                 mqd->cp_hqd_active = 1;
3645
3646         return 0;
3647 }
3648
3649 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3650 {
3651         struct amdgpu_device *adev = ring->adev;
3652         struct v9_mqd *mqd = ring->mqd_ptr;
3653         int j;
3654
3655         /* disable wptr polling */
3656         WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3657
3658         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3659                mqd->cp_hqd_eop_base_addr_lo);
3660         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3661                mqd->cp_hqd_eop_base_addr_hi);
3662
3663         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3664         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3665                mqd->cp_hqd_eop_control);
3666
3667         /* enable doorbell? */
3668         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3669                mqd->cp_hqd_pq_doorbell_control);
3670
3671         /* disable the queue if it's active */
3672         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3673                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3674                 for (j = 0; j < adev->usec_timeout; j++) {
3675                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3676                                 break;
3677                         udelay(1);
3678                 }
3679                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3680                        mqd->cp_hqd_dequeue_request);
3681                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3682                        mqd->cp_hqd_pq_rptr);
3683                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3684                        mqd->cp_hqd_pq_wptr_lo);
3685                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3686                        mqd->cp_hqd_pq_wptr_hi);
3687         }
3688
3689         /* set the pointer to the MQD */
3690         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3691                mqd->cp_mqd_base_addr_lo);
3692         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3693                mqd->cp_mqd_base_addr_hi);
3694
3695         /* set MQD vmid to 0 */
3696         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3697                mqd->cp_mqd_control);
3698
3699         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3700         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3701                mqd->cp_hqd_pq_base_lo);
3702         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3703                mqd->cp_hqd_pq_base_hi);
3704
3705         /* set up the HQD, this is similar to CP_RB0_CNTL */
3706         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3707                mqd->cp_hqd_pq_control);
3708
3709         /* set the wb address whether it's enabled or not */
3710         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3711                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3712         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3713                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3714
3715         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3716         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3717                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3718         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3719                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3720
3721         /* enable the doorbell if requested */
3722         if (ring->use_doorbell) {
3723                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3724                                         (adev->doorbell_index.kiq * 2) << 2);
3725                 /* If GC has entered CGPG, ringing doorbell > first page
3726                  * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3727                  * workaround this issue. And this change has to align with firmware
3728                  * update.
3729                  */
3730                 if (check_if_enlarge_doorbell_range(adev))
3731                         WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3732                                         (adev->doorbell.size - 4));
3733                 else
3734                         WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3735                                         (adev->doorbell_index.userqueue_end * 2) << 2);
3736         }
3737
3738         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3739                mqd->cp_hqd_pq_doorbell_control);
3740
3741         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3742         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3743                mqd->cp_hqd_pq_wptr_lo);
3744         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3745                mqd->cp_hqd_pq_wptr_hi);
3746
3747         /* set the vmid for the queue */
3748         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3749
3750         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3751                mqd->cp_hqd_persistent_state);
3752
3753         /* activate the queue */
3754         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3755                mqd->cp_hqd_active);
3756
3757         if (ring->use_doorbell)
3758                 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3759
3760         return 0;
3761 }
3762
3763 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3764 {
3765         struct amdgpu_device *adev = ring->adev;
3766         int j;
3767
3768         /* disable the queue if it's active */
3769         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3770
3771                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3772
3773                 for (j = 0; j < adev->usec_timeout; j++) {
3774                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3775                                 break;
3776                         udelay(1);
3777                 }
3778
3779                 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3780                         DRM_DEBUG("KIQ dequeue request failed.\n");
3781
3782                         /* Manual disable if dequeue request times out */
3783                         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3784                 }
3785
3786                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3787                       0);
3788         }
3789
3790         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3791         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3792         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3793         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3794         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3795         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3796         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3797         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3798
3799         return 0;
3800 }
3801
3802 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3803 {
3804         struct amdgpu_device *adev = ring->adev;
3805         struct v9_mqd *mqd = ring->mqd_ptr;
3806         struct v9_mqd *tmp_mqd;
3807
3808         gfx_v9_0_kiq_setting(ring);
3809
3810         /* GPU could be in bad state during probe, driver trigger the reset
3811          * after load the SMU, in this case , the mqd is not be initialized.
3812          * driver need to re-init the mqd.
3813          * check mqd->cp_hqd_pq_control since this value should not be 0
3814          */
3815         tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup;
3816         if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3817                 /* for GPU_RESET case , reset MQD to a clean status */
3818                 if (adev->gfx.kiq[0].mqd_backup)
3819                         memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation));
3820
3821                 /* reset ring buffer */
3822                 ring->wptr = 0;
3823                 amdgpu_ring_clear_ring(ring);
3824
3825                 mutex_lock(&adev->srbm_mutex);
3826                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3827                 gfx_v9_0_kiq_init_register(ring);
3828                 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3829                 mutex_unlock(&adev->srbm_mutex);
3830         } else {
3831                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3832                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3833                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3834                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3835                         amdgpu_ring_clear_ring(ring);
3836                 mutex_lock(&adev->srbm_mutex);
3837                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3838                 gfx_v9_0_mqd_init(ring);
3839                 gfx_v9_0_kiq_init_register(ring);
3840                 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3841                 mutex_unlock(&adev->srbm_mutex);
3842
3843                 if (adev->gfx.kiq[0].mqd_backup)
3844                         memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));
3845         }
3846
3847         return 0;
3848 }
3849
3850 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring, bool restore)
3851 {
3852         struct amdgpu_device *adev = ring->adev;
3853         struct v9_mqd *mqd = ring->mqd_ptr;
3854         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3855         struct v9_mqd *tmp_mqd;
3856
3857         /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3858          * is not be initialized before
3859          */
3860         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3861
3862         if (!restore && (!tmp_mqd->cp_hqd_pq_control ||
3863             (!amdgpu_in_reset(adev) && !adev->in_suspend))) {
3864                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3865                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3866                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3867                 mutex_lock(&adev->srbm_mutex);
3868                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3869                 gfx_v9_0_mqd_init(ring);
3870                 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3871                 mutex_unlock(&adev->srbm_mutex);
3872
3873                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3874                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3875         } else {
3876                 /* restore MQD to a clean status */
3877                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3878                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3879                 /* reset ring buffer */
3880                 ring->wptr = 0;
3881                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
3882                 amdgpu_ring_clear_ring(ring);
3883         }
3884
3885         return 0;
3886 }
3887
3888 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3889 {
3890         struct amdgpu_ring *ring;
3891         int r;
3892
3893         ring = &adev->gfx.kiq[0].ring;
3894
3895         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3896         if (unlikely(r != 0))
3897                 return r;
3898
3899         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3900         if (unlikely(r != 0)) {
3901                 amdgpu_bo_unreserve(ring->mqd_obj);
3902                 return r;
3903         }
3904
3905         gfx_v9_0_kiq_init_queue(ring);
3906         amdgpu_bo_kunmap(ring->mqd_obj);
3907         ring->mqd_ptr = NULL;
3908         amdgpu_bo_unreserve(ring->mqd_obj);
3909         return 0;
3910 }
3911
3912 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3913 {
3914         struct amdgpu_ring *ring = NULL;
3915         int r = 0, i;
3916
3917         gfx_v9_0_cp_compute_enable(adev, true);
3918
3919         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3920                 ring = &adev->gfx.compute_ring[i];
3921
3922                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3923                 if (unlikely(r != 0))
3924                         goto done;
3925                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3926                 if (!r) {
3927                         r = gfx_v9_0_kcq_init_queue(ring, false);
3928                         amdgpu_bo_kunmap(ring->mqd_obj);
3929                         ring->mqd_ptr = NULL;
3930                 }
3931                 amdgpu_bo_unreserve(ring->mqd_obj);
3932                 if (r)
3933                         goto done;
3934         }
3935
3936         r = amdgpu_gfx_enable_kcq(adev, 0);
3937 done:
3938         return r;
3939 }
3940
3941 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3942 {
3943         int r, i;
3944         struct amdgpu_ring *ring;
3945
3946         if (!(adev->flags & AMD_IS_APU))
3947                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3948
3949         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3950                 if (adev->gfx.num_gfx_rings) {
3951                         /* legacy firmware loading */
3952                         r = gfx_v9_0_cp_gfx_load_microcode(adev);
3953                         if (r)
3954                                 return r;
3955                 }
3956
3957                 r = gfx_v9_0_cp_compute_load_microcode(adev);
3958                 if (r)
3959                         return r;
3960         }
3961
3962         if (adev->gfx.num_gfx_rings)
3963                 gfx_v9_0_cp_gfx_enable(adev, false);
3964         gfx_v9_0_cp_compute_enable(adev, false);
3965
3966         r = gfx_v9_0_kiq_resume(adev);
3967         if (r)
3968                 return r;
3969
3970         if (adev->gfx.num_gfx_rings) {
3971                 r = gfx_v9_0_cp_gfx_resume(adev);
3972                 if (r)
3973                         return r;
3974         }
3975
3976         r = gfx_v9_0_kcq_resume(adev);
3977         if (r)
3978                 return r;
3979
3980         if (adev->gfx.num_gfx_rings) {
3981                 ring = &adev->gfx.gfx_ring[0];
3982                 r = amdgpu_ring_test_helper(ring);
3983                 if (r)
3984                         return r;
3985         }
3986
3987         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3988                 ring = &adev->gfx.compute_ring[i];
3989                 amdgpu_ring_test_helper(ring);
3990         }
3991
3992         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3993
3994         return 0;
3995 }
3996
3997 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3998 {
3999         u32 tmp;
4000
4001         if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) &&
4002             amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))
4003                 return;
4004
4005         tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
4006         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
4007                                 adev->df.hash_status.hash_64k);
4008         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
4009                                 adev->df.hash_status.hash_2m);
4010         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
4011                                 adev->df.hash_status.hash_1g);
4012         WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
4013 }
4014
4015 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
4016 {
4017         if (adev->gfx.num_gfx_rings)
4018                 gfx_v9_0_cp_gfx_enable(adev, enable);
4019         gfx_v9_0_cp_compute_enable(adev, enable);
4020 }
4021
4022 static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
4023 {
4024         int r;
4025         struct amdgpu_device *adev = ip_block->adev;
4026
4027         amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
4028                                        adev->gfx.cleaner_shader_ptr);
4029
4030         if (!amdgpu_sriov_vf(adev))
4031                 gfx_v9_0_init_golden_registers(adev);
4032
4033         gfx_v9_0_constants_init(adev);
4034
4035         gfx_v9_0_init_tcp_config(adev);
4036
4037         r = adev->gfx.rlc.funcs->resume(adev);
4038         if (r)
4039                 return r;
4040
4041         r = gfx_v9_0_cp_resume(adev);
4042         if (r)
4043                 return r;
4044
4045         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4046                 gfx_v9_4_2_set_power_brake_sequence(adev);
4047
4048         return r;
4049 }
4050
4051 static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block)
4052 {
4053         struct amdgpu_device *adev = ip_block->adev;
4054
4055         if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4056                 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4057         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4058         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4059         amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
4060
4061         /* DF freeze and kcq disable will fail */
4062         if (!amdgpu_ras_intr_triggered())
4063                 /* disable KCQ to avoid CPC touch memory not valid anymore */
4064                 amdgpu_gfx_disable_kcq(adev, 0);
4065
4066         if (amdgpu_sriov_vf(adev)) {
4067                 gfx_v9_0_cp_gfx_enable(adev, false);
4068                 /* must disable polling for SRIOV when hw finished, otherwise
4069                  * CPC engine may still keep fetching WB address which is already
4070                  * invalid after sw finished and trigger DMAR reading error in
4071                  * hypervisor side.
4072                  */
4073                 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4074                 return 0;
4075         }
4076
4077         /* Use deinitialize sequence from CAIL when unbinding device from driver,
4078          * otherwise KIQ is hanging when binding back
4079          */
4080         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4081                 mutex_lock(&adev->srbm_mutex);
4082                 soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me,
4083                                 adev->gfx.kiq[0].ring.pipe,
4084                                 adev->gfx.kiq[0].ring.queue, 0, 0);
4085                 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring);
4086                 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
4087                 mutex_unlock(&adev->srbm_mutex);
4088         }
4089
4090         gfx_v9_0_cp_enable(adev, false);
4091
4092         /* Skip stopping RLC with A+A reset or when RLC controls GFX clock */
4093         if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||
4094             (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) {
4095                 dev_dbg(adev->dev, "Skipping RLC halt\n");
4096                 return 0;
4097         }
4098
4099         adev->gfx.rlc.funcs->stop(adev);
4100         return 0;
4101 }
4102
4103 static int gfx_v9_0_suspend(struct amdgpu_ip_block *ip_block)
4104 {
4105         return gfx_v9_0_hw_fini(ip_block);
4106 }
4107
4108 static int gfx_v9_0_resume(struct amdgpu_ip_block *ip_block)
4109 {
4110         return gfx_v9_0_hw_init(ip_block);
4111 }
4112
4113 static bool gfx_v9_0_is_idle(void *handle)
4114 {
4115         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4116
4117         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4118                                 GRBM_STATUS, GUI_ACTIVE))
4119                 return false;
4120         else
4121                 return true;
4122 }
4123
4124 static int gfx_v9_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
4125 {
4126         unsigned i;
4127         struct amdgpu_device *adev = ip_block->adev;
4128
4129         for (i = 0; i < adev->usec_timeout; i++) {
4130                 if (gfx_v9_0_is_idle(adev))
4131                         return 0;
4132                 udelay(1);
4133         }
4134         return -ETIMEDOUT;
4135 }
4136
4137 static int gfx_v9_0_soft_reset(struct amdgpu_ip_block *ip_block)
4138 {
4139         u32 grbm_soft_reset = 0;
4140         u32 tmp;
4141         struct amdgpu_device *adev = ip_block->adev;
4142
4143         /* GRBM_STATUS */
4144         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4145         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4146                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4147                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4148                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4149                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4150                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4151                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4152                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4153                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4154                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4155         }
4156
4157         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4158                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4159                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4160         }
4161
4162         /* GRBM_STATUS2 */
4163         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4164         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4165                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4166                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4167
4168
4169         if (grbm_soft_reset) {
4170                 /* stop the rlc */
4171                 adev->gfx.rlc.funcs->stop(adev);
4172
4173                 if (adev->gfx.num_gfx_rings)
4174                         /* Disable GFX parsing/prefetching */
4175                         gfx_v9_0_cp_gfx_enable(adev, false);
4176
4177                 /* Disable MEC parsing/prefetching */
4178                 gfx_v9_0_cp_compute_enable(adev, false);
4179
4180                 if (grbm_soft_reset) {
4181                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4182                         tmp |= grbm_soft_reset;
4183                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4184                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4185                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4186
4187                         udelay(50);
4188
4189                         tmp &= ~grbm_soft_reset;
4190                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4191                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4192                 }
4193
4194                 /* Wait a little for things to settle down */
4195                 udelay(50);
4196         }
4197         return 0;
4198 }
4199
4200 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4201 {
4202         signed long r, cnt = 0;
4203         unsigned long flags;
4204         uint32_t seq, reg_val_offs = 0;
4205         uint64_t value = 0;
4206         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
4207         struct amdgpu_ring *ring = &kiq->ring;
4208
4209         BUG_ON(!ring->funcs->emit_rreg);
4210
4211         spin_lock_irqsave(&kiq->ring_lock, flags);
4212         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
4213                 pr_err("critical bug! too many kiq readers\n");
4214                 goto failed_unlock;
4215         }
4216         amdgpu_ring_alloc(ring, 32);
4217         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4218         amdgpu_ring_write(ring, 9 |     /* src: register*/
4219                                 (5 << 8) |      /* dst: memory */
4220                                 (1 << 16) |     /* count sel */
4221                                 (1 << 20));     /* write confirm */
4222         amdgpu_ring_write(ring, 0);
4223         amdgpu_ring_write(ring, 0);
4224         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4225                                 reg_val_offs * 4));
4226         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4227                                 reg_val_offs * 4));
4228         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4229         if (r)
4230                 goto failed_undo;
4231
4232         amdgpu_ring_commit(ring);
4233         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4234
4235         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4236
4237         /* don't wait anymore for gpu reset case because this way may
4238          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4239          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4240          * never return if we keep waiting in virt_kiq_rreg, which cause
4241          * gpu_recover() hang there.
4242          *
4243          * also don't wait anymore for IRQ context
4244          * */
4245         if (r < 1 && (amdgpu_in_reset(adev)))
4246                 goto failed_kiq_read;
4247
4248         might_sleep();
4249         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4250                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4251                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4252         }
4253
4254         if (cnt > MAX_KIQ_REG_TRY)
4255                 goto failed_kiq_read;
4256
4257         mb();
4258         value = (uint64_t)adev->wb.wb[reg_val_offs] |
4259                 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4260         amdgpu_device_wb_free(adev, reg_val_offs);
4261         return value;
4262
4263 failed_undo:
4264         amdgpu_ring_undo(ring);
4265 failed_unlock:
4266         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4267 failed_kiq_read:
4268         if (reg_val_offs)
4269                 amdgpu_device_wb_free(adev, reg_val_offs);
4270         pr_err("failed to read gpu clock\n");
4271         return ~0;
4272 }
4273
4274 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4275 {
4276         uint64_t clock, clock_lo, clock_hi, hi_check;
4277
4278         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
4279         case IP_VERSION(9, 3, 0):
4280                 preempt_disable();
4281                 clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4282                 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4283                 hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4284                 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
4285                  * roughly every 42 seconds.
4286                  */
4287                 if (hi_check != clock_hi) {
4288                         clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4289                         clock_hi = hi_check;
4290                 }
4291                 preempt_enable();
4292                 clock = clock_lo | (clock_hi << 32ULL);
4293                 break;
4294         default:
4295                 amdgpu_gfx_off_ctrl(adev, false);
4296                 mutex_lock(&adev->gfx.gpu_clock_mutex);
4297                 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
4298                             IP_VERSION(9, 0, 1) &&
4299                     amdgpu_sriov_runtime(adev)) {
4300                         clock = gfx_v9_0_kiq_read_clock(adev);
4301                 } else {
4302                         WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4303                         clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4304                                 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4305                 }
4306                 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4307                 amdgpu_gfx_off_ctrl(adev, true);
4308                 break;
4309         }
4310         return clock;
4311 }
4312
4313 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4314                                           uint32_t vmid,
4315                                           uint32_t gds_base, uint32_t gds_size,
4316                                           uint32_t gws_base, uint32_t gws_size,
4317                                           uint32_t oa_base, uint32_t oa_size)
4318 {
4319         struct amdgpu_device *adev = ring->adev;
4320
4321         /* GDS Base */
4322         gfx_v9_0_write_data_to_reg(ring, 0, false,
4323                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4324                                    gds_base);
4325
4326         /* GDS Size */
4327         gfx_v9_0_write_data_to_reg(ring, 0, false,
4328                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4329                                    gds_size);
4330
4331         /* GWS */
4332         gfx_v9_0_write_data_to_reg(ring, 0, false,
4333                                    SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4334                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4335
4336         /* OA */
4337         gfx_v9_0_write_data_to_reg(ring, 0, false,
4338                                    SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4339                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
4340 }
4341
4342 static const u32 vgpr_init_compute_shader[] =
4343 {
4344         0xb07c0000, 0xbe8000ff,
4345         0x000000f8, 0xbf110800,
4346         0x7e000280, 0x7e020280,
4347         0x7e040280, 0x7e060280,
4348         0x7e080280, 0x7e0a0280,
4349         0x7e0c0280, 0x7e0e0280,
4350         0x80808800, 0xbe803200,
4351         0xbf84fff5, 0xbf9c0000,
4352         0xd28c0001, 0x0001007f,
4353         0xd28d0001, 0x0002027e,
4354         0x10020288, 0xb8810904,
4355         0xb7814000, 0xd1196a01,
4356         0x00000301, 0xbe800087,
4357         0xbefc00c1, 0xd89c4000,
4358         0x00020201, 0xd89cc080,
4359         0x00040401, 0x320202ff,
4360         0x00000800, 0x80808100,
4361         0xbf84fff8, 0x7e020280,
4362         0xbf810000, 0x00000000,
4363 };
4364
4365 static const u32 sgpr_init_compute_shader[] =
4366 {
4367         0xb07c0000, 0xbe8000ff,
4368         0x0000005f, 0xbee50080,
4369         0xbe812c65, 0xbe822c65,
4370         0xbe832c65, 0xbe842c65,
4371         0xbe852c65, 0xb77c0005,
4372         0x80808500, 0xbf84fff8,
4373         0xbe800080, 0xbf810000,
4374 };
4375
4376 static const u32 vgpr_init_compute_shader_arcturus[] = {
4377         0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4378         0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4379         0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4380         0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4381         0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4382         0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4383         0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4384         0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4385         0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4386         0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4387         0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4388         0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4389         0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4390         0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4391         0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4392         0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4393         0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4394         0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4395         0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4396         0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4397         0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4398         0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4399         0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4400         0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4401         0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4402         0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4403         0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4404         0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4405         0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4406         0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4407         0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4408         0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4409         0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4410         0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4411         0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4412         0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4413         0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4414         0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4415         0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4416         0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4417         0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4418         0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4419         0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4420         0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4421         0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4422         0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4423         0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4424         0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4425         0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4426         0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4427         0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4428         0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4429         0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4430         0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4431         0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4432         0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4433         0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4434         0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4435         0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4436         0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4437         0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4438         0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4439         0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4440         0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4441         0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4442         0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4443         0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4444         0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4445         0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4446         0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4447         0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4448         0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4449         0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4450         0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4451         0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4452         0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4453         0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4454         0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4455         0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4456         0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4457         0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4458         0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4459         0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4460         0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4461         0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4462         0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4463         0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4464         0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4465         0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4466         0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4467         0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4468         0xbf84fff8, 0xbf810000,
4469 };
4470
4471 /* When below register arrays changed, please update gpr_reg_size,
4472   and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4473   to cover all gfx9 ASICs */
4474 static const struct soc15_reg_entry vgpr_init_regs[] = {
4475    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4476    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4477    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4478    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4479    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4480    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4481    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4482    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4483    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4484    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4485    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4486    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4487    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4488    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4489 };
4490
4491 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4492    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4493    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4494    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4495    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4496    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4497    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4498    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4499    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4500    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4501    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4502    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4503    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4504    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4505    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4506 };
4507
4508 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4509    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4510    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4511    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4512    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4513    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4514    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4515    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4516    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4517    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4518    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4519    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4520    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4521    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4522    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4523 };
4524
4525 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4526    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4527    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4528    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4529    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4530    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4531    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4532    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4533    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4534    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4535    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4536    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4537    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4538    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4539    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4540 };
4541
4542 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4543    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4544    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4545    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4546    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4547    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4548    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4549    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4550    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4551    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4552    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4553    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4554    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4555    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4556    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4557    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4558    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4559    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4560    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4561    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4562    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4563    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4564    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4565    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4566    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4567    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4568    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4569    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4570    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4571    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4572    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4573    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4574    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4575    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4576 };
4577
4578 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4579 {
4580         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4581         int i, r;
4582
4583         /* only support when RAS is enabled */
4584         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4585                 return 0;
4586
4587         r = amdgpu_ring_alloc(ring, 7);
4588         if (r) {
4589                 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4590                         ring->name, r);
4591                 return r;
4592         }
4593
4594         WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4595         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4596
4597         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4598         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4599                                 PACKET3_DMA_DATA_DST_SEL(1) |
4600                                 PACKET3_DMA_DATA_SRC_SEL(2) |
4601                                 PACKET3_DMA_DATA_ENGINE(0)));
4602         amdgpu_ring_write(ring, 0);
4603         amdgpu_ring_write(ring, 0);
4604         amdgpu_ring_write(ring, 0);
4605         amdgpu_ring_write(ring, 0);
4606         amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4607                                 adev->gds.gds_size);
4608
4609         amdgpu_ring_commit(ring);
4610
4611         for (i = 0; i < adev->usec_timeout; i++) {
4612                 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4613                         break;
4614                 udelay(1);
4615         }
4616
4617         if (i >= adev->usec_timeout)
4618                 r = -ETIMEDOUT;
4619
4620         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4621
4622         return r;
4623 }
4624
4625 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4626 {
4627         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4628         struct amdgpu_ib ib;
4629         struct dma_fence *f = NULL;
4630         int r, i;
4631         unsigned total_size, vgpr_offset, sgpr_offset;
4632         u64 gpu_addr;
4633
4634         int compute_dim_x = adev->gfx.config.max_shader_engines *
4635                                                 adev->gfx.config.max_cu_per_sh *
4636                                                 adev->gfx.config.max_sh_per_se;
4637         int sgpr_work_group_size = 5;
4638         int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4639         int vgpr_init_shader_size;
4640         const u32 *vgpr_init_shader_ptr;
4641         const struct soc15_reg_entry *vgpr_init_regs_ptr;
4642
4643         /* only support when RAS is enabled */
4644         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4645                 return 0;
4646
4647         /* bail if the compute ring is not ready */
4648         if (!ring->sched.ready)
4649                 return 0;
4650
4651         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) {
4652                 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4653                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4654                 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4655         } else {
4656                 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4657                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4658                 vgpr_init_regs_ptr = vgpr_init_regs;
4659         }
4660
4661         total_size =
4662                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4663         total_size +=
4664                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4665         total_size +=
4666                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4667         total_size = ALIGN(total_size, 256);
4668         vgpr_offset = total_size;
4669         total_size += ALIGN(vgpr_init_shader_size, 256);
4670         sgpr_offset = total_size;
4671         total_size += sizeof(sgpr_init_compute_shader);
4672
4673         /* allocate an indirect buffer to put the commands in */
4674         memset(&ib, 0, sizeof(ib));
4675         r = amdgpu_ib_get(adev, NULL, total_size,
4676                                         AMDGPU_IB_POOL_DIRECT, &ib);
4677         if (r) {
4678                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4679                 return r;
4680         }
4681
4682         /* load the compute shaders */
4683         for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4684                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4685
4686         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4687                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4688
4689         /* init the ib length to 0 */
4690         ib.length_dw = 0;
4691
4692         /* VGPR */
4693         /* write the register state for the compute dispatch */
4694         for (i = 0; i < gpr_reg_size; i++) {
4695                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4696                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4697                                                                 - PACKET3_SET_SH_REG_START;
4698                 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4699         }
4700         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4701         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4702         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4703         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4704                                                         - PACKET3_SET_SH_REG_START;
4705         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4706         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4707
4708         /* write dispatch packet */
4709         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4710         ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4711         ib.ptr[ib.length_dw++] = 1; /* y */
4712         ib.ptr[ib.length_dw++] = 1; /* z */
4713         ib.ptr[ib.length_dw++] =
4714                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4715
4716         /* write CS partial flush packet */
4717         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4718         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4719
4720         /* SGPR1 */
4721         /* write the register state for the compute dispatch */
4722         for (i = 0; i < gpr_reg_size; i++) {
4723                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4724                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4725                                                                 - PACKET3_SET_SH_REG_START;
4726                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4727         }
4728         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4729         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4730         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4731         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4732                                                         - PACKET3_SET_SH_REG_START;
4733         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4734         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4735
4736         /* write dispatch packet */
4737         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4738         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4739         ib.ptr[ib.length_dw++] = 1; /* y */
4740         ib.ptr[ib.length_dw++] = 1; /* z */
4741         ib.ptr[ib.length_dw++] =
4742                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4743
4744         /* write CS partial flush packet */
4745         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4746         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4747
4748         /* SGPR2 */
4749         /* write the register state for the compute dispatch */
4750         for (i = 0; i < gpr_reg_size; i++) {
4751                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4752                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4753                                                                 - PACKET3_SET_SH_REG_START;
4754                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4755         }
4756         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4757         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4758         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4759         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4760                                                         - PACKET3_SET_SH_REG_START;
4761         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4762         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4763
4764         /* write dispatch packet */
4765         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4766         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4767         ib.ptr[ib.length_dw++] = 1; /* y */
4768         ib.ptr[ib.length_dw++] = 1; /* z */
4769         ib.ptr[ib.length_dw++] =
4770                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4771
4772         /* write CS partial flush packet */
4773         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4774         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4775
4776         /* shedule the ib on the ring */
4777         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4778         if (r) {
4779                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4780                 goto fail;
4781         }
4782
4783         /* wait for the GPU to finish processing the IB */
4784         r = dma_fence_wait(f, false);
4785         if (r) {
4786                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4787                 goto fail;
4788         }
4789
4790 fail:
4791         amdgpu_ib_free(&ib, NULL);
4792         dma_fence_put(f);
4793
4794         return r;
4795 }
4796
4797 static int gfx_v9_0_early_init(struct amdgpu_ip_block *ip_block)
4798 {
4799         struct amdgpu_device *adev = ip_block->adev;
4800
4801         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
4802
4803         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
4804             amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4805                 adev->gfx.num_gfx_rings = 0;
4806         else
4807                 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4808         adev->gfx.xcc_mask = 1;
4809         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4810                                           AMDGPU_MAX_COMPUTE_RINGS);
4811         gfx_v9_0_set_kiq_pm4_funcs(adev);
4812         gfx_v9_0_set_ring_funcs(adev);
4813         gfx_v9_0_set_irq_funcs(adev);
4814         gfx_v9_0_set_gds_init(adev);
4815         gfx_v9_0_set_rlc_funcs(adev);
4816
4817         /* init rlcg reg access ctrl */
4818         gfx_v9_0_init_rlcg_reg_access_ctrl(adev);
4819
4820         return gfx_v9_0_init_microcode(adev);
4821 }
4822
4823 static int gfx_v9_0_ecc_late_init(struct amdgpu_ip_block *ip_block)
4824 {
4825         struct amdgpu_device *adev = ip_block->adev;
4826         int r;
4827
4828         /*
4829          * Temp workaround to fix the issue that CP firmware fails to
4830          * update read pointer when CPDMA is writing clearing operation
4831          * to GDS in suspend/resume sequence on several cards. So just
4832          * limit this operation in cold boot sequence.
4833          */
4834         if ((!adev->in_suspend) &&
4835             (adev->gds.gds_size)) {
4836                 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4837                 if (r)
4838                         return r;
4839         }
4840
4841         /* requires IBs so do in late init after IB pool is initialized */
4842         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4843                 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4844         else
4845                 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4846
4847         if (r)
4848                 return r;
4849
4850         if (adev->gfx.ras &&
4851             adev->gfx.ras->enable_watchdog_timer)
4852                 adev->gfx.ras->enable_watchdog_timer(adev);
4853
4854         return 0;
4855 }
4856
4857 static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block)
4858 {
4859         struct amdgpu_device *adev = ip_block->adev;
4860         int r;
4861
4862         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4863         if (r)
4864                 return r;
4865
4866         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4867         if (r)
4868                 return r;
4869
4870         r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
4871         if (r)
4872                 return r;
4873
4874         r = gfx_v9_0_ecc_late_init(ip_block);
4875         if (r)
4876                 return r;
4877
4878         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4879                 gfx_v9_4_2_debug_trap_config_init(adev,
4880                         adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4881         else
4882                 gfx_v9_0_debug_trap_config_init(adev,
4883                         adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4884
4885         return 0;
4886 }
4887
4888 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4889 {
4890         uint32_t rlc_setting;
4891
4892         /* if RLC is not enabled, do nothing */
4893         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4894         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4895                 return false;
4896
4897         return true;
4898 }
4899
4900 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
4901 {
4902         uint32_t data;
4903         unsigned i;
4904
4905         data = RLC_SAFE_MODE__CMD_MASK;
4906         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4907         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4908
4909         /* wait for RLC_SAFE_MODE */
4910         for (i = 0; i < adev->usec_timeout; i++) {
4911                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4912                         break;
4913                 udelay(1);
4914         }
4915 }
4916
4917 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
4918 {
4919         uint32_t data;
4920
4921         data = RLC_SAFE_MODE__CMD_MASK;
4922         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4923 }
4924
4925 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4926                                                 bool enable)
4927 {
4928         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4929
4930         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4931                 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4932                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4933                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4934         } else {
4935                 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4936                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4937                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4938         }
4939
4940         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4941 }
4942
4943 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4944                                                 bool enable)
4945 {
4946         /* TODO: double check if we need to perform under safe mode */
4947         /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4948
4949         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4950                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4951         else
4952                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4953
4954         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4955                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4956         else
4957                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4958
4959         /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4960 }
4961
4962 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4963                                                       bool enable)
4964 {
4965         uint32_t data, def;
4966
4967         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4968
4969         /* It is disabled by HW by default */
4970         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4971                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4972                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4973
4974                 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
4975                         data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4976
4977                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4978                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4979                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4980
4981                 /* only for Vega10 & Raven1 */
4982                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4983
4984                 if (def != data)
4985                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4986
4987                 /* MGLS is a global flag to control all MGLS in GFX */
4988                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4989                         /* 2 - RLC memory Light sleep */
4990                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4991                                 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4992                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4993                                 if (def != data)
4994                                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4995                         }
4996                         /* 3 - CP memory Light sleep */
4997                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4998                                 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4999                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5000                                 if (def != data)
5001                                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
5002                         }
5003                 }
5004         } else {
5005                 /* 1 - MGCG_OVERRIDE */
5006                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5007
5008                 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
5009                         data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
5010
5011                 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5012                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5013                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
5014                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
5015
5016                 if (def != data)
5017                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5018
5019                 /* 2 - disable MGLS in RLC */
5020                 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
5021                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
5022                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
5023                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
5024                 }
5025
5026                 /* 3 - disable MGLS in CP */
5027                 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
5028                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
5029                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5030                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
5031                 }
5032         }
5033
5034         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5035 }
5036
5037 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
5038                                            bool enable)
5039 {
5040         uint32_t data, def;
5041
5042         if (!adev->gfx.num_gfx_rings)
5043                 return;
5044
5045         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5046
5047         /* Enable 3D CGCG/CGLS */
5048         if (enable) {
5049                 /* write cmd to clear cgcg/cgls ov */
5050                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5051                 /* unset CGCG override */
5052                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5053                 /* update CGCG and CGLS override bits */
5054                 if (def != data)
5055                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5056
5057                 /* enable 3Dcgcg FSM(0x0000363f) */
5058                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5059
5060                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5061                         data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5062                                 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5063                 else
5064                         data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
5065
5066                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5067                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5068                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5069                 if (def != data)
5070                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5071
5072                 /* set IDLE_POLL_COUNT(0x00900100) */
5073                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5074                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5075                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5076                 if (def != data)
5077                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5078         } else {
5079                 /* Disable CGCG/CGLS */
5080                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5081                 /* disable cgcg, cgls should be disabled */
5082                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
5083                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
5084                 /* disable cgcg and cgls in FSM */
5085                 if (def != data)
5086                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5087         }
5088
5089         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5090 }
5091
5092 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5093                                                       bool enable)
5094 {
5095         uint32_t def, data;
5096
5097         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5098
5099         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5100                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5101                 /* unset CGCG override */
5102                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5103                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5104                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5105                 else
5106                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5107                 /* update CGCG and CGLS override bits */
5108                 if (def != data)
5109                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5110
5111                 /* enable cgcg FSM(0x0000363F) */
5112                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5113
5114                 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1))
5115                         data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5116                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5117                 else
5118                         data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5119                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5120                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5121                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5122                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5123                 if (def != data)
5124                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5125
5126                 /* set IDLE_POLL_COUNT(0x00900100) */
5127                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5128                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5129                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5130                 if (def != data)
5131                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5132         } else {
5133                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5134                 /* reset CGCG/CGLS bits */
5135                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5136                 /* disable cgcg and cgls in FSM */
5137                 if (def != data)
5138                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5139         }
5140
5141         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5142 }
5143
5144 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5145                                             bool enable)
5146 {
5147         if (enable) {
5148                 /* CGCG/CGLS should be enabled after MGCG/MGLS
5149                  * ===  MGCG + MGLS ===
5150                  */
5151                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5152                 /* ===  CGCG /CGLS for GFX 3D Only === */
5153                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5154                 /* ===  CGCG + CGLS === */
5155                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5156         } else {
5157                 /* CGCG/CGLS should be disabled before MGCG/MGLS
5158                  * ===  CGCG + CGLS ===
5159                  */
5160                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5161                 /* ===  CGCG /CGLS for GFX 3D Only === */
5162                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5163                 /* ===  MGCG + MGLS === */
5164                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5165         }
5166         return 0;
5167 }
5168
5169 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
5170                                               unsigned int vmid)
5171 {
5172         u32 reg, data;
5173
5174         reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5175         if (amdgpu_sriov_is_pp_one_vf(adev))
5176                 data = RREG32_NO_KIQ(reg);
5177         else
5178                 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);
5179
5180         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5181         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5182
5183         if (amdgpu_sriov_is_pp_one_vf(adev))
5184                 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5185         else
5186                 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5187 }
5188
5189 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid)
5190 {
5191         amdgpu_gfx_off_ctrl(adev, false);
5192
5193         gfx_v9_0_update_spm_vmid_internal(adev, vmid);
5194
5195         amdgpu_gfx_off_ctrl(adev, true);
5196 }
5197
5198 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5199                                         uint32_t offset,
5200                                         struct soc15_reg_rlcg *entries, int arr_size)
5201 {
5202         int i;
5203         uint32_t reg;
5204
5205         if (!entries)
5206                 return false;
5207
5208         for (i = 0; i < arr_size; i++) {
5209                 const struct soc15_reg_rlcg *entry;
5210
5211                 entry = &entries[i];
5212                 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5213                 if (offset == reg)
5214                         return true;
5215         }
5216
5217         return false;
5218 }
5219
5220 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5221 {
5222         return gfx_v9_0_check_rlcg_range(adev, offset,
5223                                         (void *)rlcg_access_gc_9_0,
5224                                         ARRAY_SIZE(rlcg_access_gc_9_0));
5225 }
5226
5227 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5228         .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5229         .set_safe_mode = gfx_v9_0_set_safe_mode,
5230         .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5231         .init = gfx_v9_0_rlc_init,
5232         .get_csb_size = gfx_v9_0_get_csb_size,
5233         .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5234         .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5235         .resume = gfx_v9_0_rlc_resume,
5236         .stop = gfx_v9_0_rlc_stop,
5237         .reset = gfx_v9_0_rlc_reset,
5238         .start = gfx_v9_0_rlc_start,
5239         .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5240         .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5241 };
5242
5243 static int gfx_v9_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
5244                                           enum amd_powergating_state state)
5245 {
5246         struct amdgpu_device *adev = ip_block->adev;
5247         bool enable = (state == AMD_PG_STATE_GATE);
5248
5249         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5250         case IP_VERSION(9, 2, 2):
5251         case IP_VERSION(9, 1, 0):
5252         case IP_VERSION(9, 3, 0):
5253                 if (!enable)
5254                         amdgpu_gfx_off_ctrl(adev, false);
5255
5256                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5257                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5258                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5259                 } else {
5260                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5261                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5262                 }
5263
5264                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5265                         gfx_v9_0_enable_cp_power_gating(adev, true);
5266                 else
5267                         gfx_v9_0_enable_cp_power_gating(adev, false);
5268
5269                 /* update gfx cgpg state */
5270                 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5271
5272                 /* update mgcg state */
5273                 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5274
5275                 if (enable)
5276                         amdgpu_gfx_off_ctrl(adev, true);
5277                 break;
5278         case IP_VERSION(9, 2, 1):
5279                 amdgpu_gfx_off_ctrl(adev, enable);
5280                 break;
5281         default:
5282                 break;
5283         }
5284
5285         return 0;
5286 }
5287
5288 static int gfx_v9_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
5289                                           enum amd_clockgating_state state)
5290 {
5291         struct amdgpu_device *adev = ip_block->adev;
5292
5293         if (amdgpu_sriov_vf(adev))
5294                 return 0;
5295
5296         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5297         case IP_VERSION(9, 0, 1):
5298         case IP_VERSION(9, 2, 1):
5299         case IP_VERSION(9, 4, 0):
5300         case IP_VERSION(9, 2, 2):
5301         case IP_VERSION(9, 1, 0):
5302         case IP_VERSION(9, 4, 1):
5303         case IP_VERSION(9, 3, 0):
5304         case IP_VERSION(9, 4, 2):
5305                 gfx_v9_0_update_gfx_clock_gating(adev,
5306                                                  state == AMD_CG_STATE_GATE);
5307                 break;
5308         default:
5309                 break;
5310         }
5311         return 0;
5312 }
5313
5314 static void gfx_v9_0_get_clockgating_state(void *handle, u64 *flags)
5315 {
5316         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5317         int data;
5318
5319         if (amdgpu_sriov_vf(adev))
5320                 *flags = 0;
5321
5322         /* AMD_CG_SUPPORT_GFX_MGCG */
5323         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5324         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5325                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5326
5327         /* AMD_CG_SUPPORT_GFX_CGCG */
5328         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5329         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5330                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5331
5332         /* AMD_CG_SUPPORT_GFX_CGLS */
5333         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5334                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5335
5336         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5337         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5338         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5339                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5340
5341         /* AMD_CG_SUPPORT_GFX_CP_LS */
5342         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5343         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5344                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5345
5346         if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) {
5347                 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5348                 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5349                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5350                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5351
5352                 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5353                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5354                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5355         }
5356 }
5357
5358 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5359 {
5360         return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/
5361 }
5362
5363 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5364 {
5365         struct amdgpu_device *adev = ring->adev;
5366         u64 wptr;
5367
5368         /* XXX check if swapping is necessary on BE */
5369         if (ring->use_doorbell) {
5370                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5371         } else {
5372                 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5373                 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5374         }
5375
5376         return wptr;
5377 }
5378
5379 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5380 {
5381         struct amdgpu_device *adev = ring->adev;
5382
5383         if (ring->use_doorbell) {
5384                 /* XXX check if swapping is necessary on BE */
5385                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5386                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5387         } else {
5388                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5389                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5390         }
5391 }
5392
5393 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5394 {
5395         struct amdgpu_device *adev = ring->adev;
5396         u32 ref_and_mask, reg_mem_engine;
5397         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5398
5399         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5400                 switch (ring->me) {
5401                 case 1:
5402                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5403                         break;
5404                 case 2:
5405                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5406                         break;
5407                 default:
5408                         return;
5409                 }
5410                 reg_mem_engine = 0;
5411         } else {
5412                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5413                 reg_mem_engine = 1; /* pfp */
5414         }
5415
5416         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5417                               adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5418                               adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5419                               ref_and_mask, ref_and_mask, 0x20);
5420 }
5421
5422 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5423                                         struct amdgpu_job *job,
5424                                         struct amdgpu_ib *ib,
5425                                         uint32_t flags)
5426 {
5427         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5428         u32 header, control = 0;
5429
5430         if (ib->flags & AMDGPU_IB_FLAG_CE)
5431                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5432         else
5433                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5434
5435         control |= ib->length_dw | (vmid << 24);
5436
5437         if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
5438                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5439
5440                 if (flags & AMDGPU_IB_PREEMPTED)
5441                         control |= INDIRECT_BUFFER_PRE_RESUME(1);
5442
5443                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5444                         gfx_v9_0_ring_emit_de_meta(ring,
5445                                                    (!amdgpu_sriov_vf(ring->adev) &&
5446                                                    flags & AMDGPU_IB_PREEMPTED) ?
5447                                                    true : false,
5448                                                    job->gds_size > 0 && job->gds_base != 0);
5449         }
5450
5451         amdgpu_ring_write(ring, header);
5452         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5453         amdgpu_ring_write(ring,
5454 #ifdef __BIG_ENDIAN
5455                 (2 << 0) |
5456 #endif
5457                 lower_32_bits(ib->gpu_addr));
5458         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5459         amdgpu_ring_ib_on_emit_cntl(ring);
5460         amdgpu_ring_write(ring, control);
5461 }
5462
5463 static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,
5464                                      unsigned offset)
5465 {
5466         u32 control = ring->ring[offset];
5467
5468         control |= INDIRECT_BUFFER_PRE_RESUME(1);
5469         ring->ring[offset] = control;
5470 }
5471
5472 static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,
5473                                         unsigned offset)
5474 {
5475         struct amdgpu_device *adev = ring->adev;
5476         void *ce_payload_cpu_addr;
5477         uint64_t payload_offset, payload_size;
5478
5479         payload_size = sizeof(struct v9_ce_ib_state);
5480
5481         if (ring->is_mes_queue) {
5482                 payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5483                                           gfx[0].gfx_meta_data) +
5484                         offsetof(struct v9_gfx_meta_data, ce_payload);
5485                 ce_payload_cpu_addr =
5486                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);
5487         } else {
5488                 payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5489                 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5490         }
5491
5492         if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5493                 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size);
5494         } else {
5495                 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr,
5496                        (ring->buf_mask + 1 - offset) << 2);
5497                 payload_size -= (ring->buf_mask + 1 - offset) << 2;
5498                 memcpy((void *)&ring->ring[0],
5499                        ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5500                        payload_size);
5501         }
5502 }
5503
5504 static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,
5505                                         unsigned offset)
5506 {
5507         struct amdgpu_device *adev = ring->adev;
5508         void *de_payload_cpu_addr;
5509         uint64_t payload_offset, payload_size;
5510
5511         payload_size = sizeof(struct v9_de_ib_state);
5512
5513         if (ring->is_mes_queue) {
5514                 payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5515                                           gfx[0].gfx_meta_data) +
5516                         offsetof(struct v9_gfx_meta_data, de_payload);
5517                 de_payload_cpu_addr =
5518                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);
5519         } else {
5520                 payload_offset = offsetof(struct v9_gfx_meta_data, de_payload);
5521                 de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5522         }
5523
5524         ((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status =
5525                 IB_COMPLETION_STATUS_PREEMPTED;
5526
5527         if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5528                 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size);
5529         } else {
5530                 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr,
5531                        (ring->buf_mask + 1 - offset) << 2);
5532                 payload_size -= (ring->buf_mask + 1 - offset) << 2;
5533                 memcpy((void *)&ring->ring[0],
5534                        de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5535                        payload_size);
5536         }
5537 }
5538
5539 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5540                                           struct amdgpu_job *job,
5541                                           struct amdgpu_ib *ib,
5542                                           uint32_t flags)
5543 {
5544         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5545         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5546
5547         /* Currently, there is a high possibility to get wave ID mismatch
5548          * between ME and GDS, leading to a hw deadlock, because ME generates
5549          * different wave IDs than the GDS expects. This situation happens
5550          * randomly when at least 5 compute pipes use GDS ordered append.
5551          * The wave IDs generated by ME are also wrong after suspend/resume.
5552          * Those are probably bugs somewhere else in the kernel driver.
5553          *
5554          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5555          * GDS to 0 for this ring (me/pipe).
5556          */
5557         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5558                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5559                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5560                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5561         }
5562
5563         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5564         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5565         amdgpu_ring_write(ring,
5566 #ifdef __BIG_ENDIAN
5567                                 (2 << 0) |
5568 #endif
5569                                 lower_32_bits(ib->gpu_addr));
5570         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5571         amdgpu_ring_write(ring, control);
5572 }
5573
5574 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5575                                      u64 seq, unsigned flags)
5576 {
5577         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5578         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5579         bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5580         bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;
5581         uint32_t dw2 = 0;
5582
5583         /* RELEASE_MEM - flush caches, send int */
5584         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5585
5586         if (writeback) {
5587                 dw2 = EOP_TC_NC_ACTION_EN;
5588         } else {
5589                 dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN |
5590                                 EOP_TC_MD_ACTION_EN;
5591         }
5592         dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5593                                 EVENT_INDEX(5);
5594         if (exec)
5595                 dw2 |= EOP_EXEC;
5596
5597         amdgpu_ring_write(ring, dw2);
5598         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5599
5600         /*
5601          * the address should be Qword aligned if 64bit write, Dword
5602          * aligned if only send 32bit data low (discard data high)
5603          */
5604         if (write64bit)
5605                 BUG_ON(addr & 0x7);
5606         else
5607                 BUG_ON(addr & 0x3);
5608         amdgpu_ring_write(ring, lower_32_bits(addr));
5609         amdgpu_ring_write(ring, upper_32_bits(addr));
5610         amdgpu_ring_write(ring, lower_32_bits(seq));
5611         amdgpu_ring_write(ring, upper_32_bits(seq));
5612         amdgpu_ring_write(ring, 0);
5613 }
5614
5615 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5616 {
5617         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5618         uint32_t seq = ring->fence_drv.sync_seq;
5619         uint64_t addr = ring->fence_drv.gpu_addr;
5620
5621         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5622                               lower_32_bits(addr), upper_32_bits(addr),
5623                               seq, 0xffffffff, 4);
5624 }
5625
5626 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5627                                         unsigned vmid, uint64_t pd_addr)
5628 {
5629         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5630
5631         /* compute doesn't have PFP */
5632         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5633                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5634                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5635                 amdgpu_ring_write(ring, 0x0);
5636         }
5637 }
5638
5639 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5640 {
5641         return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */
5642 }
5643
5644 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5645 {
5646         u64 wptr;
5647
5648         /* XXX check if swapping is necessary on BE */
5649         if (ring->use_doorbell)
5650                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5651         else
5652                 BUG();
5653         return wptr;
5654 }
5655
5656 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5657 {
5658         struct amdgpu_device *adev = ring->adev;
5659
5660         /* XXX check if swapping is necessary on BE */
5661         if (ring->use_doorbell) {
5662                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5663                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5664         } else{
5665                 BUG(); /* only DOORBELL method supported on gfx9 now */
5666         }
5667 }
5668
5669 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5670                                          u64 seq, unsigned int flags)
5671 {
5672         struct amdgpu_device *adev = ring->adev;
5673
5674         /* we only allocate 32bit for each seq wb address */
5675         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5676
5677         /* write fence seq to the "addr" */
5678         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5679         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5680                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5681         amdgpu_ring_write(ring, lower_32_bits(addr));
5682         amdgpu_ring_write(ring, upper_32_bits(addr));
5683         amdgpu_ring_write(ring, lower_32_bits(seq));
5684
5685         if (flags & AMDGPU_FENCE_FLAG_INT) {
5686                 /* set register to trigger INT */
5687                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5688                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5689                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5690                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5691                 amdgpu_ring_write(ring, 0);
5692                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5693         }
5694 }
5695
5696 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5697 {
5698         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5699         amdgpu_ring_write(ring, 0);
5700 }
5701
5702 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
5703 {
5704         struct amdgpu_device *adev = ring->adev;
5705         struct v9_ce_ib_state ce_payload = {0};
5706         uint64_t offset, ce_payload_gpu_addr;
5707         void *ce_payload_cpu_addr;
5708         int cnt;
5709
5710         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5711
5712         if (ring->is_mes_queue) {
5713                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5714                                   gfx[0].gfx_meta_data) +
5715                         offsetof(struct v9_gfx_meta_data, ce_payload);
5716                 ce_payload_gpu_addr =
5717                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5718                 ce_payload_cpu_addr =
5719                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5720         } else {
5721                 offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5722                 ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5723                 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5724         }
5725
5726         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5727         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5728                                  WRITE_DATA_DST_SEL(8) |
5729                                  WR_CONFIRM) |
5730                                  WRITE_DATA_CACHE_POLICY(0));
5731         amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));
5732         amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));
5733
5734         amdgpu_ring_ib_on_emit_ce(ring);
5735
5736         if (resume)
5737                 amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr,
5738                                            sizeof(ce_payload) >> 2);
5739         else
5740                 amdgpu_ring_write_multiple(ring, (void *)&ce_payload,
5741                                            sizeof(ce_payload) >> 2);
5742 }
5743
5744 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)
5745 {
5746         int i, r = 0;
5747         struct amdgpu_device *adev = ring->adev;
5748         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5749         struct amdgpu_ring *kiq_ring = &kiq->ring;
5750         unsigned long flags;
5751
5752         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5753                 return -EINVAL;
5754
5755         spin_lock_irqsave(&kiq->ring_lock, flags);
5756
5757         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5758                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5759                 return -ENOMEM;
5760         }
5761
5762         /* assert preemption condition */
5763         amdgpu_ring_set_preempt_cond_exec(ring, false);
5764
5765         ring->trail_seq += 1;
5766         amdgpu_ring_alloc(ring, 13);
5767         gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
5768                                  ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);
5769
5770         /* assert IB preemption, emit the trailing fence */
5771         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5772                                    ring->trail_fence_gpu_addr,
5773                                    ring->trail_seq);
5774
5775         amdgpu_ring_commit(kiq_ring);
5776         spin_unlock_irqrestore(&kiq->ring_lock, flags);
5777
5778         /* poll the trailing fence */
5779         for (i = 0; i < adev->usec_timeout; i++) {
5780                 if (ring->trail_seq ==
5781                         le32_to_cpu(*ring->trail_fence_cpu_addr))
5782                         break;
5783                 udelay(1);
5784         }
5785
5786         if (i >= adev->usec_timeout) {
5787                 r = -EINVAL;
5788                 DRM_WARN("ring %d timeout to preempt ib\n", ring->idx);
5789         }
5790
5791         /*reset the CP_VMID_PREEMPT after trailing fence*/
5792         amdgpu_ring_emit_wreg(ring,
5793                               SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT),
5794                               0x0);
5795         amdgpu_ring_commit(ring);
5796
5797         /* deassert preemption condition */
5798         amdgpu_ring_set_preempt_cond_exec(ring, true);
5799         return r;
5800 }
5801
5802 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)
5803 {
5804         struct amdgpu_device *adev = ring->adev;
5805         struct v9_de_ib_state de_payload = {0};
5806         uint64_t offset, gds_addr, de_payload_gpu_addr;
5807         void *de_payload_cpu_addr;
5808         int cnt;
5809
5810         if (ring->is_mes_queue) {
5811                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5812                                   gfx[0].gfx_meta_data) +
5813                         offsetof(struct v9_gfx_meta_data, de_payload);
5814                 de_payload_gpu_addr =
5815                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5816                 de_payload_cpu_addr =
5817                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5818
5819                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5820                                   gfx[0].gds_backup) +
5821                         offsetof(struct v9_gfx_meta_data, de_payload);
5822                 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5823         } else {
5824                 offset = offsetof(struct v9_gfx_meta_data, de_payload);
5825                 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5826                 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5827
5828                 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5829                                  AMDGPU_CSA_SIZE - adev->gds.gds_size,
5830                                  PAGE_SIZE);
5831         }
5832
5833         if (usegds) {
5834                 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5835                 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5836         }
5837
5838         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5839         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5840         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5841                                  WRITE_DATA_DST_SEL(8) |
5842                                  WR_CONFIRM) |
5843                                  WRITE_DATA_CACHE_POLICY(0));
5844         amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5845         amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5846
5847         amdgpu_ring_ib_on_emit_de(ring);
5848         if (resume)
5849                 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5850                                            sizeof(de_payload) >> 2);
5851         else
5852                 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5853                                            sizeof(de_payload) >> 2);
5854 }
5855
5856 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5857                                    bool secure)
5858 {
5859         uint32_t v = secure ? FRAME_TMZ : 0;
5860
5861         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5862         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5863 }
5864
5865 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5866 {
5867         uint32_t dw2 = 0;
5868
5869         gfx_v9_0_ring_emit_ce_meta(ring,
5870                                    (!amdgpu_sriov_vf(ring->adev) &&
5871                                    flags & AMDGPU_IB_PREEMPTED) ? true : false);
5872
5873         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5874         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5875                 /* set load_global_config & load_global_uconfig */
5876                 dw2 |= 0x8001;
5877                 /* set load_cs_sh_regs */
5878                 dw2 |= 0x01000000;
5879                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5880                 dw2 |= 0x10002;
5881
5882                 /* set load_ce_ram if preamble presented */
5883                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5884                         dw2 |= 0x10000000;
5885         } else {
5886                 /* still load_ce_ram if this is the first time preamble presented
5887                  * although there is no context switch happens.
5888                  */
5889                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5890                         dw2 |= 0x10000000;
5891         }
5892
5893         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5894         amdgpu_ring_write(ring, dw2);
5895         amdgpu_ring_write(ring, 0);
5896 }
5897
5898 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5899                                                   uint64_t addr)
5900 {
5901         unsigned ret;
5902         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5903         amdgpu_ring_write(ring, lower_32_bits(addr));
5904         amdgpu_ring_write(ring, upper_32_bits(addr));
5905         /* discard following DWs if *cond_exec_gpu_addr==0 */
5906         amdgpu_ring_write(ring, 0);
5907         ret = ring->wptr & ring->buf_mask;
5908         /* patch dummy value later */
5909         amdgpu_ring_write(ring, 0);
5910         return ret;
5911 }
5912
5913 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5914                                     uint32_t reg_val_offs)
5915 {
5916         struct amdgpu_device *adev = ring->adev;
5917
5918         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5919         amdgpu_ring_write(ring, 0 |     /* src: register*/
5920                                 (5 << 8) |      /* dst: memory */
5921                                 (1 << 20));     /* write confirm */
5922         amdgpu_ring_write(ring, reg);
5923         amdgpu_ring_write(ring, 0);
5924         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5925                                 reg_val_offs * 4));
5926         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5927                                 reg_val_offs * 4));
5928 }
5929
5930 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5931                                     uint32_t val)
5932 {
5933         uint32_t cmd = 0;
5934
5935         switch (ring->funcs->type) {
5936         case AMDGPU_RING_TYPE_GFX:
5937                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5938                 break;
5939         case AMDGPU_RING_TYPE_KIQ:
5940                 cmd = (1 << 16); /* no inc addr */
5941                 break;
5942         default:
5943                 cmd = WR_CONFIRM;
5944                 break;
5945         }
5946         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5947         amdgpu_ring_write(ring, cmd);
5948         amdgpu_ring_write(ring, reg);
5949         amdgpu_ring_write(ring, 0);
5950         amdgpu_ring_write(ring, val);
5951 }
5952
5953 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5954                                         uint32_t val, uint32_t mask)
5955 {
5956         gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5957 }
5958
5959 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5960                                                   uint32_t reg0, uint32_t reg1,
5961                                                   uint32_t ref, uint32_t mask)
5962 {
5963         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5964         struct amdgpu_device *adev = ring->adev;
5965         bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5966                 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5967
5968         if (fw_version_ok)
5969                 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5970                                       ref, mask, 0x20);
5971         else
5972                 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5973                                                            ref, mask);
5974 }
5975
5976 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5977 {
5978         struct amdgpu_device *adev = ring->adev;
5979         uint32_t value = 0;
5980
5981         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5982         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5983         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5984         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5985         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5986         WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5987         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5988 }
5989
5990 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5991                                                  enum amdgpu_interrupt_state state)
5992 {
5993         switch (state) {
5994         case AMDGPU_IRQ_STATE_DISABLE:
5995         case AMDGPU_IRQ_STATE_ENABLE:
5996                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5997                                TIME_STAMP_INT_ENABLE,
5998                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5999                 break;
6000         default:
6001                 break;
6002         }
6003 }
6004
6005 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6006                                                      int me, int pipe,
6007                                                      enum amdgpu_interrupt_state state)
6008 {
6009         u32 mec_int_cntl, mec_int_cntl_reg;
6010
6011         /*
6012          * amdgpu controls only the first MEC. That's why this function only
6013          * handles the setting of interrupts for this specific MEC. All other
6014          * pipes' interrupts are set by amdkfd.
6015          */
6016
6017         if (me == 1) {
6018                 switch (pipe) {
6019                 case 0:
6020                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
6021                         break;
6022                 case 1:
6023                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
6024                         break;
6025                 case 2:
6026                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
6027                         break;
6028                 case 3:
6029                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
6030                         break;
6031                 default:
6032                         DRM_DEBUG("invalid pipe %d\n", pipe);
6033                         return;
6034                 }
6035         } else {
6036                 DRM_DEBUG("invalid me %d\n", me);
6037                 return;
6038         }
6039
6040         switch (state) {
6041         case AMDGPU_IRQ_STATE_DISABLE:
6042                 mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg);
6043                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6044                                              TIME_STAMP_INT_ENABLE, 0);
6045                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6046                 break;
6047         case AMDGPU_IRQ_STATE_ENABLE:
6048                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6049                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6050                                              TIME_STAMP_INT_ENABLE, 1);
6051                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6052                 break;
6053         default:
6054                 break;
6055         }
6056 }
6057
6058 static u32 gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device *adev,
6059                                      int me, int pipe)
6060 {
6061         /*
6062          * amdgpu controls only the first MEC. That's why this function only
6063          * handles the setting of interrupts for this specific MEC. All other
6064          * pipes' interrupts are set by amdkfd.
6065          */
6066         if (me != 1)
6067                 return 0;
6068
6069         switch (pipe) {
6070         case 0:
6071                 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
6072         case 1:
6073                 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
6074         case 2:
6075                 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
6076         case 3:
6077                 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
6078         default:
6079                 return 0;
6080         }
6081 }
6082
6083 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6084                                              struct amdgpu_irq_src *source,
6085                                              unsigned type,
6086                                              enum amdgpu_interrupt_state state)
6087 {
6088         u32 cp_int_cntl_reg, cp_int_cntl;
6089         int i, j;
6090
6091         switch (state) {
6092         case AMDGPU_IRQ_STATE_DISABLE:
6093         case AMDGPU_IRQ_STATE_ENABLE:
6094                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6095                                PRIV_REG_INT_ENABLE,
6096                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6097                 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6098                         for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6099                                 /* MECs start at 1 */
6100                                 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6101
6102                                 if (cp_int_cntl_reg) {
6103                                         cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6104                                         cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6105                                                                     PRIV_REG_INT_ENABLE,
6106                                                                     state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6107                                         WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6108                                 }
6109                         }
6110                 }
6111                 break;
6112         default:
6113                 break;
6114         }
6115
6116         return 0;
6117 }
6118
6119 static int gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6120                                            struct amdgpu_irq_src *source,
6121                                            unsigned type,
6122                                            enum amdgpu_interrupt_state state)
6123 {
6124         u32 cp_int_cntl_reg, cp_int_cntl;
6125         int i, j;
6126
6127         switch (state) {
6128         case AMDGPU_IRQ_STATE_DISABLE:
6129         case AMDGPU_IRQ_STATE_ENABLE:
6130                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6131                                OPCODE_ERROR_INT_ENABLE,
6132                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6133                 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6134                         for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6135                                 /* MECs start at 1 */
6136                                 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6137
6138                                 if (cp_int_cntl_reg) {
6139                                         cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6140                                         cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6141                                                                     OPCODE_ERROR_INT_ENABLE,
6142                                                                     state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6143                                         WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6144                                 }
6145                         }
6146                 }
6147                 break;
6148         default:
6149                 break;
6150         }
6151
6152         return 0;
6153 }
6154
6155 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6156                                               struct amdgpu_irq_src *source,
6157                                               unsigned type,
6158                                               enum amdgpu_interrupt_state state)
6159 {
6160         switch (state) {
6161         case AMDGPU_IRQ_STATE_DISABLE:
6162         case AMDGPU_IRQ_STATE_ENABLE:
6163                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6164                                PRIV_INSTR_INT_ENABLE,
6165                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6166                 break;
6167         default:
6168                 break;
6169         }
6170
6171         return 0;
6172 }
6173
6174 #define ENABLE_ECC_ON_ME_PIPE(me, pipe)                         \
6175         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6176                         CP_ECC_ERROR_INT_ENABLE, 1)
6177
6178 #define DISABLE_ECC_ON_ME_PIPE(me, pipe)                        \
6179         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6180                         CP_ECC_ERROR_INT_ENABLE, 0)
6181
6182 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
6183                                               struct amdgpu_irq_src *source,
6184                                               unsigned type,
6185                                               enum amdgpu_interrupt_state state)
6186 {
6187         switch (state) {
6188         case AMDGPU_IRQ_STATE_DISABLE:
6189                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6190                                 CP_ECC_ERROR_INT_ENABLE, 0);
6191                 DISABLE_ECC_ON_ME_PIPE(1, 0);
6192                 DISABLE_ECC_ON_ME_PIPE(1, 1);
6193                 DISABLE_ECC_ON_ME_PIPE(1, 2);
6194                 DISABLE_ECC_ON_ME_PIPE(1, 3);
6195                 break;
6196
6197         case AMDGPU_IRQ_STATE_ENABLE:
6198                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6199                                 CP_ECC_ERROR_INT_ENABLE, 1);
6200                 ENABLE_ECC_ON_ME_PIPE(1, 0);
6201                 ENABLE_ECC_ON_ME_PIPE(1, 1);
6202                 ENABLE_ECC_ON_ME_PIPE(1, 2);
6203                 ENABLE_ECC_ON_ME_PIPE(1, 3);
6204                 break;
6205         default:
6206                 break;
6207         }
6208
6209         return 0;
6210 }
6211
6212
6213 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6214                                             struct amdgpu_irq_src *src,
6215                                             unsigned type,
6216                                             enum amdgpu_interrupt_state state)
6217 {
6218         switch (type) {
6219         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6220                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
6221                 break;
6222         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6223                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6224                 break;
6225         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6226                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6227                 break;
6228         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6229                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6230                 break;
6231         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6232                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6233                 break;
6234         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6235                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6236                 break;
6237         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6238                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6239                 break;
6240         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6241                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6242                 break;
6243         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6244                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6245                 break;
6246         default:
6247                 break;
6248         }
6249         return 0;
6250 }
6251
6252 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
6253                             struct amdgpu_irq_src *source,
6254                             struct amdgpu_iv_entry *entry)
6255 {
6256         int i;
6257         u8 me_id, pipe_id, queue_id;
6258         struct amdgpu_ring *ring;
6259
6260         DRM_DEBUG("IH: CP EOP\n");
6261         me_id = (entry->ring_id & 0x0c) >> 2;
6262         pipe_id = (entry->ring_id & 0x03) >> 0;
6263         queue_id = (entry->ring_id & 0x70) >> 4;
6264
6265         switch (me_id) {
6266         case 0:
6267                 if (adev->gfx.num_gfx_rings) {
6268                         if (!adev->gfx.mcbp) {
6269                                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6270                         } else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) {
6271                                 /* Fence signals are handled on the software rings*/
6272                                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
6273                                         amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]);
6274                         }
6275                 }
6276                 break;
6277         case 1:
6278         case 2:
6279                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6280                         ring = &adev->gfx.compute_ring[i];
6281                         /* Per-queue interrupt is supported for MEC starting from VI.
6282                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
6283                           */
6284                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6285                                 amdgpu_fence_process(ring);
6286                 }
6287                 break;
6288         }
6289         return 0;
6290 }
6291
6292 static void gfx_v9_0_fault(struct amdgpu_device *adev,
6293                            struct amdgpu_iv_entry *entry)
6294 {
6295         u8 me_id, pipe_id, queue_id;
6296         struct amdgpu_ring *ring;
6297         int i;
6298
6299         me_id = (entry->ring_id & 0x0c) >> 2;
6300         pipe_id = (entry->ring_id & 0x03) >> 0;
6301         queue_id = (entry->ring_id & 0x70) >> 4;
6302
6303         switch (me_id) {
6304         case 0:
6305                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
6306                 break;
6307         case 1:
6308         case 2:
6309                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6310                         ring = &adev->gfx.compute_ring[i];
6311                         if (ring->me == me_id && ring->pipe == pipe_id &&
6312                             ring->queue == queue_id)
6313                                 drm_sched_fault(&ring->sched);
6314                 }
6315                 break;
6316         }
6317 }
6318
6319 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
6320                                  struct amdgpu_irq_src *source,
6321                                  struct amdgpu_iv_entry *entry)
6322 {
6323         DRM_ERROR("Illegal register access in command stream\n");
6324         gfx_v9_0_fault(adev, entry);
6325         return 0;
6326 }
6327
6328 static int gfx_v9_0_bad_op_irq(struct amdgpu_device *adev,
6329                                struct amdgpu_irq_src *source,
6330                                struct amdgpu_iv_entry *entry)
6331 {
6332         DRM_ERROR("Illegal opcode in command stream\n");
6333         gfx_v9_0_fault(adev, entry);
6334         return 0;
6335 }
6336
6337 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
6338                                   struct amdgpu_irq_src *source,
6339                                   struct amdgpu_iv_entry *entry)
6340 {
6341         DRM_ERROR("Illegal instruction in command stream\n");
6342         gfx_v9_0_fault(adev, entry);
6343         return 0;
6344 }
6345
6346
6347 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
6348         { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
6349           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
6350           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
6351         },
6352         { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
6353           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
6354           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
6355         },
6356         { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6357           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
6358           0, 0
6359         },
6360         { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6361           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
6362           0, 0
6363         },
6364         { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
6365           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
6366           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
6367         },
6368         { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6369           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
6370           0, 0
6371         },
6372         { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6373           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
6374           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
6375         },
6376         { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
6377           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
6378           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
6379         },
6380         { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6381           SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
6382           0, 0
6383         },
6384         { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6385           SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
6386           0, 0
6387         },
6388         { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6389           SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
6390           0, 0
6391         },
6392         { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6393           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
6394           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
6395         },
6396         { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6397           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
6398           0, 0
6399         },
6400         { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6401           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6402           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6403         },
6404         { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6405           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6406           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6407           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6408         },
6409         { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6410           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6411           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6412           0, 0
6413         },
6414         { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6415           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6416           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6417           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6418         },
6419         { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6420           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6421           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6422           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6423         },
6424         { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6425           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6426           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6427           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6428         },
6429         { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6430           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6431           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6432           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6433         },
6434         { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6435           SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6436           0, 0
6437         },
6438         { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6439           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6440           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6441         },
6442         { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6443           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6444           0, 0
6445         },
6446         { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6447           SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6448           0, 0
6449         },
6450         { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6451           SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6452           0, 0
6453         },
6454         { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6455           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6456           0, 0
6457         },
6458         { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6459           SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6460           0, 0
6461         },
6462         { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6463           SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6464           0, 0
6465         },
6466         { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6467           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6468           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6469         },
6470         { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6471           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6472           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6473         },
6474         { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6475           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6476           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6477         },
6478         { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6479           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6480           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6481         },
6482         { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6483           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6484           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6485         },
6486         { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6487           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6488           0, 0
6489         },
6490         { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6491           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6492           0, 0
6493         },
6494         { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6495           SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6496           0, 0
6497         },
6498         { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6499           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6500           0, 0
6501         },
6502         { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6503           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6504           0, 0
6505         },
6506         { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6507           SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6508           0, 0
6509         },
6510         { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6511           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6512           0, 0
6513         },
6514         { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6515           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6516           0, 0
6517         },
6518         { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6519           SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6520           0, 0
6521         },
6522         { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6523           SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6524           0, 0
6525         },
6526         { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6527           SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6528           0, 0
6529         },
6530         { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6531           SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6532           0, 0
6533         },
6534         { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6535           SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6536           0, 0
6537         },
6538         { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6539           SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6540           0, 0
6541         },
6542         { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6543           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6544           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6545         },
6546         { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6547           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6548           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6549         },
6550         { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6551           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6552           0, 0
6553         },
6554         { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6555           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6556           0, 0
6557         },
6558         { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6559           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6560           0, 0
6561         },
6562         { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6563           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6564           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6565         },
6566         { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6567           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6568           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6569         },
6570         { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6571           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6572           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6573         },
6574         { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6575           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6576           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6577         },
6578         { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6579           SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6580           0, 0
6581         },
6582         { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6583           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6584           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6585         },
6586         { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6587           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6588           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6589         },
6590         { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6591           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6592           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6593         },
6594         { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6595           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6596           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6597         },
6598         { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6599           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6600           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6601         },
6602         { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6603           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6604           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6605         },
6606         { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6607           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6608           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6609         },
6610         { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6611           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6612           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6613         },
6614         { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6615           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6616           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6617         },
6618         { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6619           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6620           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6621         },
6622         { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6623           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6624           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6625         },
6626         { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6627           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6628           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6629         },
6630         { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6631           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6632           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6633         },
6634         { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6635           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6636           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6637         },
6638         { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6639           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6640           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6641         },
6642         { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6643           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6644           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6645         },
6646         { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6647           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6648           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6649         },
6650         { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6651           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6652           0, 0
6653         },
6654         { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6655           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6656           0, 0
6657         },
6658         { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6659           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6660           0, 0
6661         },
6662         { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6663           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6664           0, 0
6665         },
6666         { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6667           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6668           0, 0
6669         },
6670         { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6671           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6672           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6673         },
6674         { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6675           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6676           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6677         },
6678         { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6679           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6680           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6681         },
6682         { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6683           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6684           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6685         },
6686         { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6687           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6688           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6689         },
6690         { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6691           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6692           0, 0
6693         },
6694         { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6695           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6696           0, 0
6697         },
6698         { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6699           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6700           0, 0
6701         },
6702         { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6703           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6704           0, 0
6705         },
6706         { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6707           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6708           0, 0
6709         },
6710         { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6711           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6712           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6713         },
6714         { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6715           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6716           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6717         },
6718         { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6719           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6720           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6721         },
6722         { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6723           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6724           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6725         },
6726         { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6727           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6728           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6729         },
6730         { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6731           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6732           0, 0
6733         },
6734         { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6735           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6736           0, 0
6737         },
6738         { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6739           SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6740           0, 0
6741         },
6742         { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6743           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6744           0, 0
6745         },
6746         { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6747           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6748           0, 0
6749         },
6750         { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6751           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6752           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6753         },
6754         { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6755           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6756           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6757         },
6758         { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6759           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6760           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6761         },
6762         { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6763           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6764           0, 0
6765         },
6766         { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6767           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6768           0, 0
6769         },
6770         { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6771           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6772           0, 0
6773         },
6774         { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6775           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6776           0, 0
6777         },
6778         { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6779           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6780           0, 0
6781         },
6782         { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6783           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6784           0, 0
6785         }
6786 };
6787
6788 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6789                                      void *inject_if, uint32_t instance_mask)
6790 {
6791         struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6792         int ret;
6793         struct ta_ras_trigger_error_input block_info = { 0 };
6794
6795         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6796                 return -EINVAL;
6797
6798         if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6799                 return -EINVAL;
6800
6801         if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6802                 return -EPERM;
6803
6804         if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6805               info->head.type)) {
6806                 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6807                         ras_gfx_subblocks[info->head.sub_block_index].name,
6808                         info->head.type);
6809                 return -EPERM;
6810         }
6811
6812         if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6813               info->head.type)) {
6814                 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6815                         ras_gfx_subblocks[info->head.sub_block_index].name,
6816                         info->head.type);
6817                 return -EPERM;
6818         }
6819
6820         block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6821         block_info.sub_block_index =
6822                 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6823         block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6824         block_info.address = info->address;
6825         block_info.value = info->value;
6826
6827         mutex_lock(&adev->grbm_idx_mutex);
6828         ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask);
6829         mutex_unlock(&adev->grbm_idx_mutex);
6830
6831         return ret;
6832 }
6833
6834 static const char * const vml2_mems[] = {
6835         "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6836         "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6837         "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6838         "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6839         "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6840         "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6841         "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6842         "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6843         "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6844         "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6845         "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6846         "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6847         "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6848         "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6849         "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6850         "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6851 };
6852
6853 static const char * const vml2_walker_mems[] = {
6854         "UTC_VML2_CACHE_PDE0_MEM0",
6855         "UTC_VML2_CACHE_PDE0_MEM1",
6856         "UTC_VML2_CACHE_PDE1_MEM0",
6857         "UTC_VML2_CACHE_PDE1_MEM1",
6858         "UTC_VML2_CACHE_PDE2_MEM0",
6859         "UTC_VML2_CACHE_PDE2_MEM1",
6860         "UTC_VML2_RDIF_LOG_FIFO",
6861 };
6862
6863 static const char * const atc_l2_cache_2m_mems[] = {
6864         "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6865         "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6866         "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6867         "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6868 };
6869
6870 static const char *atc_l2_cache_4k_mems[] = {
6871         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6872         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6873         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6874         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6875         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6876         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6877         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6878         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6879         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6880         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6881         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6882         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6883         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6884         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6885         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6886         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6887         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6888         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6889         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6890         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6891         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6892         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6893         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6894         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6895         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6896         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6897         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6898         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6899         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6900         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6901         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6902         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6903 };
6904
6905 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6906                                          struct ras_err_data *err_data)
6907 {
6908         uint32_t i, data;
6909         uint32_t sec_count, ded_count;
6910
6911         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6912         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6913         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6914         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6915         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6916         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6917         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6918         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6919
6920         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6921                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6922                 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6923
6924                 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6925                 if (sec_count) {
6926                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6927                                 "SEC %d\n", i, vml2_mems[i], sec_count);
6928                         err_data->ce_count += sec_count;
6929                 }
6930
6931                 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6932                 if (ded_count) {
6933                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6934                                 "DED %d\n", i, vml2_mems[i], ded_count);
6935                         err_data->ue_count += ded_count;
6936                 }
6937         }
6938
6939         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6940                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6941                 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6942
6943                 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6944                                                 SEC_COUNT);
6945                 if (sec_count) {
6946                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6947                                 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6948                         err_data->ce_count += sec_count;
6949                 }
6950
6951                 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6952                                                 DED_COUNT);
6953                 if (ded_count) {
6954                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6955                                 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6956                         err_data->ue_count += ded_count;
6957                 }
6958         }
6959
6960         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6961                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6962                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6963
6964                 sec_count = (data & 0x00006000L) >> 0xd;
6965                 if (sec_count) {
6966                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6967                                 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6968                                 sec_count);
6969                         err_data->ce_count += sec_count;
6970                 }
6971         }
6972
6973         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6974                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6975                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6976
6977                 sec_count = (data & 0x00006000L) >> 0xd;
6978                 if (sec_count) {
6979                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6980                                 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6981                                 sec_count);
6982                         err_data->ce_count += sec_count;
6983                 }
6984
6985                 ded_count = (data & 0x00018000L) >> 0xf;
6986                 if (ded_count) {
6987                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6988                                 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6989                                 ded_count);
6990                         err_data->ue_count += ded_count;
6991                 }
6992         }
6993
6994         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6995         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6996         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6997         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6998
6999         return 0;
7000 }
7001
7002 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
7003         const struct soc15_reg_entry *reg,
7004         uint32_t se_id, uint32_t inst_id, uint32_t value,
7005         uint32_t *sec_count, uint32_t *ded_count)
7006 {
7007         uint32_t i;
7008         uint32_t sec_cnt, ded_cnt;
7009
7010         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
7011                 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
7012                         gfx_v9_0_ras_fields[i].seg != reg->seg ||
7013                         gfx_v9_0_ras_fields[i].inst != reg->inst)
7014                         continue;
7015
7016                 sec_cnt = (value &
7017                                 gfx_v9_0_ras_fields[i].sec_count_mask) >>
7018                                 gfx_v9_0_ras_fields[i].sec_count_shift;
7019                 if (sec_cnt) {
7020                         dev_info(adev->dev, "GFX SubBlock %s, "
7021                                 "Instance[%d][%d], SEC %d\n",
7022                                 gfx_v9_0_ras_fields[i].name,
7023                                 se_id, inst_id,
7024                                 sec_cnt);
7025                         *sec_count += sec_cnt;
7026                 }
7027
7028                 ded_cnt = (value &
7029                                 gfx_v9_0_ras_fields[i].ded_count_mask) >>
7030                                 gfx_v9_0_ras_fields[i].ded_count_shift;
7031                 if (ded_cnt) {
7032                         dev_info(adev->dev, "GFX SubBlock %s, "
7033                                 "Instance[%d][%d], DED %d\n",
7034                                 gfx_v9_0_ras_fields[i].name,
7035                                 se_id, inst_id,
7036                                 ded_cnt);
7037                         *ded_count += ded_cnt;
7038                 }
7039         }
7040
7041         return 0;
7042 }
7043
7044 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
7045 {
7046         int i, j, k;
7047
7048         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
7049                 return;
7050
7051         /* read back registers to clear the counters */
7052         mutex_lock(&adev->grbm_idx_mutex);
7053         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
7054                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
7055                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7056                                 amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0);
7057                                 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7058                         }
7059                 }
7060         }
7061         WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
7062         mutex_unlock(&adev->grbm_idx_mutex);
7063
7064         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7065         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
7066         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7067         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
7068         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7069         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
7070         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7071         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
7072
7073         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
7074                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
7075                 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
7076         }
7077
7078         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
7079                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
7080                 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
7081         }
7082
7083         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
7084                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
7085                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
7086         }
7087
7088         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
7089                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
7090                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
7091         }
7092
7093         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7094         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7095         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7096         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7097 }
7098
7099 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
7100                                           void *ras_error_status)
7101 {
7102         struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
7103         uint32_t sec_count = 0, ded_count = 0;
7104         uint32_t i, j, k;
7105         uint32_t reg_value;
7106
7107         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
7108                 return;
7109
7110         err_data->ue_count = 0;
7111         err_data->ce_count = 0;
7112
7113         mutex_lock(&adev->grbm_idx_mutex);
7114
7115         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
7116                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
7117                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7118                                 amdgpu_gfx_select_se_sh(adev, j, 0, k, 0);
7119                                 reg_value =
7120                                         RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7121                                 if (reg_value)
7122                                         gfx_v9_0_ras_error_count(adev,
7123                                                 &gfx_v9_0_edc_counter_regs[i],
7124                                                 j, k, reg_value,
7125                                                 &sec_count, &ded_count);
7126                         }
7127                 }
7128         }
7129
7130         err_data->ce_count += sec_count;
7131         err_data->ue_count += ded_count;
7132
7133         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7134         mutex_unlock(&adev->grbm_idx_mutex);
7135
7136         gfx_v9_0_query_utc_edc_status(adev, err_data);
7137 }
7138
7139 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
7140 {
7141         const unsigned int cp_coher_cntl =
7142                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
7143                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
7144                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
7145                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
7146                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
7147
7148         /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
7149         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
7150         amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
7151         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
7152         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
7153         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
7154         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
7155         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
7156 }
7157
7158 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
7159                                         uint32_t pipe, bool enable)
7160 {
7161         struct amdgpu_device *adev = ring->adev;
7162         uint32_t val;
7163         uint32_t wcl_cs_reg;
7164
7165         /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
7166         val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
7167
7168         switch (pipe) {
7169         case 0:
7170                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
7171                 break;
7172         case 1:
7173                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
7174                 break;
7175         case 2:
7176                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
7177                 break;
7178         case 3:
7179                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
7180                 break;
7181         default:
7182                 DRM_DEBUG("invalid pipe %d\n", pipe);
7183                 return;
7184         }
7185
7186         amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
7187
7188 }
7189 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
7190 {
7191         struct amdgpu_device *adev = ring->adev;
7192         uint32_t val;
7193         int i;
7194
7195
7196         /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
7197          * number of gfx waves. Setting 5 bit will make sure gfx only gets
7198          * around 25% of gpu resources.
7199          */
7200         val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
7201         amdgpu_ring_emit_wreg(ring,
7202                               SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
7203                               val);
7204
7205         /* Restrict waves for normal/low priority compute queues as well
7206          * to get best QoS for high priority compute jobs.
7207          *
7208          * amdgpu controls only 1st ME(0-3 CS pipes).
7209          */
7210         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
7211                 if (i != ring->pipe)
7212                         gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
7213
7214         }
7215 }
7216
7217 static void gfx_v9_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
7218 {
7219         /* Header itself is a NOP packet */
7220         if (num_nop == 1) {
7221                 amdgpu_ring_write(ring, ring->funcs->nop);
7222                 return;
7223         }
7224
7225         /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
7226         amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
7227
7228         /* Header is at index 0, followed by num_nops - 1 NOP packet's */
7229         amdgpu_ring_insert_nop(ring, num_nop - 1);
7230 }
7231
7232 static int gfx_v9_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid)
7233 {
7234         struct amdgpu_device *adev = ring->adev;
7235         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7236         struct amdgpu_ring *kiq_ring = &kiq->ring;
7237         unsigned long flags;
7238         u32 tmp;
7239         int r;
7240
7241         if (amdgpu_sriov_vf(adev))
7242                 return -EINVAL;
7243
7244         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
7245                 return -EINVAL;
7246
7247         spin_lock_irqsave(&kiq->ring_lock, flags);
7248
7249         if (amdgpu_ring_alloc(kiq_ring, 5)) {
7250                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7251                 return -ENOMEM;
7252         }
7253
7254         tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
7255         gfx_v9_0_ring_emit_wreg(kiq_ring,
7256                                  SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), tmp);
7257         amdgpu_ring_commit(kiq_ring);
7258
7259         spin_unlock_irqrestore(&kiq->ring_lock, flags);
7260
7261         r = amdgpu_ring_test_ring(kiq_ring);
7262         if (r)
7263                 return r;
7264
7265         if (amdgpu_ring_alloc(ring, 7 + 7 + 5))
7266                 return -ENOMEM;
7267         gfx_v9_0_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
7268                                  ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC);
7269         gfx_v9_0_ring_emit_reg_wait(ring,
7270                                     SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0, 0xffff);
7271         gfx_v9_0_ring_emit_wreg(ring,
7272                                 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0);
7273
7274         return amdgpu_ring_test_ring(ring);
7275 }
7276
7277 static int gfx_v9_0_reset_kcq(struct amdgpu_ring *ring,
7278                               unsigned int vmid)
7279 {
7280         struct amdgpu_device *adev = ring->adev;
7281         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7282         struct amdgpu_ring *kiq_ring = &kiq->ring;
7283         unsigned long flags;
7284         int i, r;
7285
7286         if (amdgpu_sriov_vf(adev))
7287                 return -EINVAL;
7288
7289         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
7290                 return -EINVAL;
7291
7292         spin_lock_irqsave(&kiq->ring_lock, flags);
7293
7294         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
7295                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7296                 return -ENOMEM;
7297         }
7298
7299         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
7300                                    0, 0);
7301         amdgpu_ring_commit(kiq_ring);
7302
7303         spin_unlock_irqrestore(&kiq->ring_lock, flags);
7304
7305         r = amdgpu_ring_test_ring(kiq_ring);
7306         if (r)
7307                 return r;
7308
7309         /* make sure dequeue is complete*/
7310         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
7311         mutex_lock(&adev->srbm_mutex);
7312         soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
7313         for (i = 0; i < adev->usec_timeout; i++) {
7314                 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
7315                         break;
7316                 udelay(1);
7317         }
7318         if (i >= adev->usec_timeout)
7319                 r = -ETIMEDOUT;
7320         soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7321         mutex_unlock(&adev->srbm_mutex);
7322         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
7323         if (r) {
7324                 dev_err(adev->dev, "fail to wait on hqd deactive\n");
7325                 return r;
7326         }
7327
7328         r = amdgpu_bo_reserve(ring->mqd_obj, false);
7329         if (unlikely(r != 0)){
7330                 dev_err(adev->dev, "fail to resv mqd_obj\n");
7331                 return r;
7332         }
7333         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
7334         if (!r) {
7335                 r = gfx_v9_0_kcq_init_queue(ring, true);
7336                 amdgpu_bo_kunmap(ring->mqd_obj);
7337                 ring->mqd_ptr = NULL;
7338         }
7339         amdgpu_bo_unreserve(ring->mqd_obj);
7340         if (r) {
7341                 dev_err(adev->dev, "fail to unresv mqd_obj\n");
7342                 return r;
7343         }
7344         spin_lock_irqsave(&kiq->ring_lock, flags);
7345         r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
7346         if (r) {
7347                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7348                 return -ENOMEM;
7349         }
7350         kiq->pmf->kiq_map_queues(kiq_ring, ring);
7351         amdgpu_ring_commit(kiq_ring);
7352         spin_unlock_irqrestore(&kiq->ring_lock, flags);
7353         r = amdgpu_ring_test_ring(kiq_ring);
7354         if (r) {
7355                 DRM_ERROR("fail to remap queue\n");
7356                 return r;
7357         }
7358         return amdgpu_ring_test_ring(ring);
7359 }
7360
7361 static void gfx_v9_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
7362 {
7363         struct amdgpu_device *adev = ip_block->adev;
7364         uint32_t i, j, k, reg, index = 0;
7365         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7366
7367         if (!adev->gfx.ip_dump_core)
7368                 return;
7369
7370         for (i = 0; i < reg_count; i++)
7371                 drm_printf(p, "%-50s \t 0x%08x\n",
7372                            gc_reg_list_9[i].reg_name,
7373                            adev->gfx.ip_dump_core[i]);
7374
7375         /* print compute queue registers for all instances */
7376         if (!adev->gfx.ip_dump_compute_queues)
7377                 return;
7378
7379         reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7380         drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
7381                    adev->gfx.mec.num_mec,
7382                    adev->gfx.mec.num_pipe_per_mec,
7383                    adev->gfx.mec.num_queue_per_pipe);
7384
7385         for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7386                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7387                         for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7388                                 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
7389                                 for (reg = 0; reg < reg_count; reg++) {
7390                                         drm_printf(p, "%-50s \t 0x%08x\n",
7391                                                    gc_cp_reg_list_9[reg].reg_name,
7392                                                    adev->gfx.ip_dump_compute_queues[index + reg]);
7393                                 }
7394                                 index += reg_count;
7395                         }
7396                 }
7397         }
7398
7399 }
7400
7401 static void gfx_v9_ip_dump(struct amdgpu_ip_block *ip_block)
7402 {
7403         struct amdgpu_device *adev = ip_block->adev;
7404         uint32_t i, j, k, reg, index = 0;
7405         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7406
7407         if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings)
7408                 return;
7409
7410         amdgpu_gfx_off_ctrl(adev, false);
7411         for (i = 0; i < reg_count; i++)
7412                 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i]));
7413         amdgpu_gfx_off_ctrl(adev, true);
7414
7415         /* dump compute queue registers for all instances */
7416         if (!adev->gfx.ip_dump_compute_queues)
7417                 return;
7418
7419         reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7420         amdgpu_gfx_off_ctrl(adev, false);
7421         mutex_lock(&adev->srbm_mutex);
7422         for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7423                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7424                         for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7425                                 /* ME0 is for GFX so start from 1 for CP */
7426                                 soc15_grbm_select(adev, 1 + i, j, k, 0, 0);
7427
7428                                 for (reg = 0; reg < reg_count; reg++) {
7429                                         adev->gfx.ip_dump_compute_queues[index + reg] =
7430                                                 RREG32(SOC15_REG_ENTRY_OFFSET(
7431                                                         gc_cp_reg_list_9[reg]));
7432                                 }
7433                                 index += reg_count;
7434                         }
7435                 }
7436         }
7437         soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7438         mutex_unlock(&adev->srbm_mutex);
7439         amdgpu_gfx_off_ctrl(adev, true);
7440
7441 }
7442
7443 static void gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
7444 {
7445         /* Emit the cleaner shader */
7446         amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
7447         amdgpu_ring_write(ring, 0);  /* RESERVED field, programmed to zero */
7448 }
7449
7450 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
7451         .name = "gfx_v9_0",
7452         .early_init = gfx_v9_0_early_init,
7453         .late_init = gfx_v9_0_late_init,
7454         .sw_init = gfx_v9_0_sw_init,
7455         .sw_fini = gfx_v9_0_sw_fini,
7456         .hw_init = gfx_v9_0_hw_init,
7457         .hw_fini = gfx_v9_0_hw_fini,
7458         .suspend = gfx_v9_0_suspend,
7459         .resume = gfx_v9_0_resume,
7460         .is_idle = gfx_v9_0_is_idle,
7461         .wait_for_idle = gfx_v9_0_wait_for_idle,
7462         .soft_reset = gfx_v9_0_soft_reset,
7463         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
7464         .set_powergating_state = gfx_v9_0_set_powergating_state,
7465         .get_clockgating_state = gfx_v9_0_get_clockgating_state,
7466         .dump_ip_state = gfx_v9_ip_dump,
7467         .print_ip_state = gfx_v9_ip_print,
7468 };
7469
7470 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
7471         .type = AMDGPU_RING_TYPE_GFX,
7472         .align_mask = 0xff,
7473         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7474         .support_64bit_ptrs = true,
7475         .secure_submission_supported = true,
7476         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
7477         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
7478         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
7479         .emit_frame_size = /* totally 242 maximum if 16 IBs */
7480                 5 +  /* COND_EXEC */
7481                 7 +  /* PIPELINE_SYNC */
7482                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7483                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7484                 2 + /* VM_FLUSH */
7485                 8 +  /* FENCE for VM_FLUSH */
7486                 20 + /* GDS switch */
7487                 4 + /* double SWITCH_BUFFER,
7488                        the first COND_EXEC jump to the place just
7489                            prior to this double SWITCH_BUFFER  */
7490                 5 + /* COND_EXEC */
7491                 7 +      /*     HDP_flush */
7492                 4 +      /*     VGT_flush */
7493                 14 + /* CE_META */
7494                 31 + /* DE_META */
7495                 3 + /* CNTX_CTRL */
7496                 5 + /* HDP_INVL */
7497                 8 + 8 + /* FENCE x2 */
7498                 2 + /* SWITCH_BUFFER */
7499                 7 + /* gfx_v9_0_emit_mem_sync */
7500                 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7501         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
7502         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7503         .emit_fence = gfx_v9_0_ring_emit_fence,
7504         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7505         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7506         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7507         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7508         .test_ring = gfx_v9_0_ring_test_ring,
7509         .insert_nop = gfx_v9_ring_insert_nop,
7510         .pad_ib = amdgpu_ring_generic_pad_ib,
7511         .emit_switch_buffer = gfx_v9_ring_emit_sb,
7512         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7513         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7514         .preempt_ib = gfx_v9_0_ring_preempt_ib,
7515         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7516         .emit_wreg = gfx_v9_0_ring_emit_wreg,
7517         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7518         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7519         .soft_recovery = gfx_v9_0_ring_soft_recovery,
7520         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7521         .reset = gfx_v9_0_reset_kgq,
7522         .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7523         .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7524         .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7525 };
7526
7527 static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {
7528         .type = AMDGPU_RING_TYPE_GFX,
7529         .align_mask = 0xff,
7530         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7531         .support_64bit_ptrs = true,
7532         .secure_submission_supported = true,
7533         .get_rptr = amdgpu_sw_ring_get_rptr_gfx,
7534         .get_wptr = amdgpu_sw_ring_get_wptr_gfx,
7535         .set_wptr = amdgpu_sw_ring_set_wptr_gfx,
7536         .emit_frame_size = /* totally 242 maximum if 16 IBs */
7537                 5 +  /* COND_EXEC */
7538                 7 +  /* PIPELINE_SYNC */
7539                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7540                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7541                 2 + /* VM_FLUSH */
7542                 8 +  /* FENCE for VM_FLUSH */
7543                 20 + /* GDS switch */
7544                 4 + /* double SWITCH_BUFFER,
7545                      * the first COND_EXEC jump to the place just
7546                      * prior to this double SWITCH_BUFFER
7547                      */
7548                 5 + /* COND_EXEC */
7549                 7 +      /*     HDP_flush */
7550                 4 +      /*     VGT_flush */
7551                 14 + /* CE_META */
7552                 31 + /* DE_META */
7553                 3 + /* CNTX_CTRL */
7554                 5 + /* HDP_INVL */
7555                 8 + 8 + /* FENCE x2 */
7556                 2 + /* SWITCH_BUFFER */
7557                 7 + /* gfx_v9_0_emit_mem_sync */
7558                 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7559         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
7560         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7561         .emit_fence = gfx_v9_0_ring_emit_fence,
7562         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7563         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7564         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7565         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7566         .test_ring = gfx_v9_0_ring_test_ring,
7567         .test_ib = gfx_v9_0_ring_test_ib,
7568         .insert_nop = gfx_v9_ring_insert_nop,
7569         .pad_ib = amdgpu_ring_generic_pad_ib,
7570         .emit_switch_buffer = gfx_v9_ring_emit_sb,
7571         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7572         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7573         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7574         .emit_wreg = gfx_v9_0_ring_emit_wreg,
7575         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7576         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7577         .soft_recovery = gfx_v9_0_ring_soft_recovery,
7578         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7579         .patch_cntl = gfx_v9_0_ring_patch_cntl,
7580         .patch_de = gfx_v9_0_ring_patch_de_meta,
7581         .patch_ce = gfx_v9_0_ring_patch_ce_meta,
7582         .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7583         .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7584         .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7585 };
7586
7587 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
7588         .type = AMDGPU_RING_TYPE_COMPUTE,
7589         .align_mask = 0xff,
7590         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7591         .support_64bit_ptrs = true,
7592         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
7593         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
7594         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
7595         .emit_frame_size =
7596                 20 + /* gfx_v9_0_ring_emit_gds_switch */
7597                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
7598                 5 + /* hdp invalidate */
7599                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7600                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7601                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7602                 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
7603                 7 + /* gfx_v9_0_emit_mem_sync */
7604                 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
7605                 15 + /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
7606                 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7607         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7608         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
7609         .emit_fence = gfx_v9_0_ring_emit_fence,
7610         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7611         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7612         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7613         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7614         .test_ring = gfx_v9_0_ring_test_ring,
7615         .test_ib = gfx_v9_0_ring_test_ib,
7616         .insert_nop = gfx_v9_ring_insert_nop,
7617         .pad_ib = amdgpu_ring_generic_pad_ib,
7618         .emit_wreg = gfx_v9_0_ring_emit_wreg,
7619         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7620         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7621         .soft_recovery = gfx_v9_0_ring_soft_recovery,
7622         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7623         .emit_wave_limit = gfx_v9_0_emit_wave_limit,
7624         .reset = gfx_v9_0_reset_kcq,
7625         .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7626         .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7627         .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7628 };
7629
7630 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
7631         .type = AMDGPU_RING_TYPE_KIQ,
7632         .align_mask = 0xff,
7633         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7634         .support_64bit_ptrs = true,
7635         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
7636         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
7637         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
7638         .emit_frame_size =
7639                 20 + /* gfx_v9_0_ring_emit_gds_switch */
7640                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
7641                 5 + /* hdp invalidate */
7642                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7643                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7644                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7645                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7646         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7647         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
7648         .test_ring = gfx_v9_0_ring_test_ring,
7649         .insert_nop = amdgpu_ring_insert_nop,
7650         .pad_ib = amdgpu_ring_generic_pad_ib,
7651         .emit_rreg = gfx_v9_0_ring_emit_rreg,
7652         .emit_wreg = gfx_v9_0_ring_emit_wreg,
7653         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7654         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7655 };
7656
7657 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
7658 {
7659         int i;
7660
7661         adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq;
7662
7663         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7664                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
7665
7666         if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
7667                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
7668                         adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;
7669         }
7670
7671         for (i = 0; i < adev->gfx.num_compute_rings; i++)
7672                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
7673 }
7674
7675 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
7676         .set = gfx_v9_0_set_eop_interrupt_state,
7677         .process = gfx_v9_0_eop_irq,
7678 };
7679
7680 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
7681         .set = gfx_v9_0_set_priv_reg_fault_state,
7682         .process = gfx_v9_0_priv_reg_irq,
7683 };
7684
7685 static const struct amdgpu_irq_src_funcs gfx_v9_0_bad_op_irq_funcs = {
7686         .set = gfx_v9_0_set_bad_op_fault_state,
7687         .process = gfx_v9_0_bad_op_irq,
7688 };
7689
7690 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
7691         .set = gfx_v9_0_set_priv_inst_fault_state,
7692         .process = gfx_v9_0_priv_inst_irq,
7693 };
7694
7695 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
7696         .set = gfx_v9_0_set_cp_ecc_error_state,
7697         .process = amdgpu_gfx_cp_ecc_error_irq,
7698 };
7699
7700
7701 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
7702 {
7703         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7704         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
7705
7706         adev->gfx.priv_reg_irq.num_types = 1;
7707         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
7708
7709         adev->gfx.bad_op_irq.num_types = 1;
7710         adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs;
7711
7712         adev->gfx.priv_inst_irq.num_types = 1;
7713         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7714
7715         adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7716         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7717 }
7718
7719 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7720 {
7721         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7722         case IP_VERSION(9, 0, 1):
7723         case IP_VERSION(9, 2, 1):
7724         case IP_VERSION(9, 4, 0):
7725         case IP_VERSION(9, 2, 2):
7726         case IP_VERSION(9, 1, 0):
7727         case IP_VERSION(9, 4, 1):
7728         case IP_VERSION(9, 3, 0):
7729         case IP_VERSION(9, 4, 2):
7730                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7731                 break;
7732         default:
7733                 break;
7734         }
7735 }
7736
7737 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7738 {
7739         /* init asci gds info */
7740         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7741         case IP_VERSION(9, 0, 1):
7742         case IP_VERSION(9, 2, 1):
7743         case IP_VERSION(9, 4, 0):
7744                 adev->gds.gds_size = 0x10000;
7745                 break;
7746         case IP_VERSION(9, 2, 2):
7747         case IP_VERSION(9, 1, 0):
7748         case IP_VERSION(9, 4, 1):
7749                 adev->gds.gds_size = 0x1000;
7750                 break;
7751         case IP_VERSION(9, 4, 2):
7752                 /* aldebaran removed all the GDS internal memory,
7753                  * only support GWS opcode in kernel, like barrier
7754                  * semaphore.etc */
7755                 adev->gds.gds_size = 0;
7756                 break;
7757         default:
7758                 adev->gds.gds_size = 0x10000;
7759                 break;
7760         }
7761
7762         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7763         case IP_VERSION(9, 0, 1):
7764         case IP_VERSION(9, 4, 0):
7765                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7766                 break;
7767         case IP_VERSION(9, 2, 1):
7768                 adev->gds.gds_compute_max_wave_id = 0x27f;
7769                 break;
7770         case IP_VERSION(9, 2, 2):
7771         case IP_VERSION(9, 1, 0):
7772                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7773                         adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7774                 else
7775                         adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7776                 break;
7777         case IP_VERSION(9, 4, 1):
7778                 adev->gds.gds_compute_max_wave_id = 0xfff;
7779                 break;
7780         case IP_VERSION(9, 4, 2):
7781                 /* deprecated for Aldebaran, no usage at all */
7782                 adev->gds.gds_compute_max_wave_id = 0;
7783                 break;
7784         default:
7785                 /* this really depends on the chip */
7786                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7787                 break;
7788         }
7789
7790         adev->gds.gws_size = 64;
7791         adev->gds.oa_size = 16;
7792 }
7793
7794 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7795                                                  u32 bitmap)
7796 {
7797         u32 data;
7798
7799         if (!bitmap)
7800                 return;
7801
7802         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7803         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7804
7805         WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7806 }
7807
7808 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7809 {
7810         u32 data, mask;
7811
7812         data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7813         data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7814
7815         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7816         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7817
7818         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7819
7820         return (~data) & mask;
7821 }
7822
7823 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7824                                  struct amdgpu_cu_info *cu_info)
7825 {
7826         int i, j, k, counter, active_cu_number = 0;
7827         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7828         unsigned disable_masks[4 * 4];
7829
7830         if (!adev || !cu_info)
7831                 return -EINVAL;
7832
7833         /*
7834          * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7835          */
7836         if (adev->gfx.config.max_shader_engines *
7837                 adev->gfx.config.max_sh_per_se > 16)
7838                 return -EINVAL;
7839
7840         amdgpu_gfx_parse_disable_cu(disable_masks,
7841                                     adev->gfx.config.max_shader_engines,
7842                                     adev->gfx.config.max_sh_per_se);
7843
7844         mutex_lock(&adev->grbm_idx_mutex);
7845         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7846                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7847                         mask = 1;
7848                         ao_bitmap = 0;
7849                         counter = 0;
7850                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
7851                         gfx_v9_0_set_user_cu_inactive_bitmap(
7852                                 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7853                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7854
7855                         /*
7856                          * The bitmap(and ao_cu_bitmap) in cu_info structure is
7857                          * 4x4 size array, and it's usually suitable for Vega
7858                          * ASICs which has 4*2 SE/SH layout.
7859                          * But for Arcturus, SE/SH layout is changed to 8*1.
7860                          * To mostly reduce the impact, we make it compatible
7861                          * with current bitmap array as below:
7862                          *    SE4,SH0 --> bitmap[0][1]
7863                          *    SE5,SH0 --> bitmap[1][1]
7864                          *    SE6,SH0 --> bitmap[2][1]
7865                          *    SE7,SH0 --> bitmap[3][1]
7866                          */
7867                         cu_info->bitmap[0][i % 4][j + i / 4] = bitmap;
7868
7869                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7870                                 if (bitmap & mask) {
7871                                         if (counter < adev->gfx.config.max_cu_per_sh)
7872                                                 ao_bitmap |= mask;
7873                                         counter ++;
7874                                 }
7875                                 mask <<= 1;
7876                         }
7877                         active_cu_number += counter;
7878                         if (i < 2 && j < 2)
7879                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7880                         cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7881                 }
7882         }
7883         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7884         mutex_unlock(&adev->grbm_idx_mutex);
7885
7886         cu_info->number = active_cu_number;
7887         cu_info->ao_cu_mask = ao_cu_mask;
7888         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7889
7890         return 0;
7891 }
7892
7893 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7894 {
7895         .type = AMD_IP_BLOCK_TYPE_GFX,
7896         .major = 9,
7897         .minor = 0,
7898         .rev = 0,
7899         .funcs = &gfx_v9_0_ip_funcs,
7900 };