drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "soc15.h"
  33 #include "soc15d.h"
  34 #include "amdgpu_atomfirmware.h"
  35 #include "amdgpu_pm.h"
  36
  37 #include "gc/gc_9_0_offset.h"
  38 #include "gc/gc_9_0_sh_mask.h"
  39
  40 #include "vega10_enum.h"
  41
  42 #include "soc15_common.h"
  43 #include "clearstate_gfx9.h"
  44 #include "v9_structs.h"
  45
  46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
  47
  48 #include "amdgpu_ras.h"
  49
  50 #include "amdgpu_ring_mux.h"
  51 #include "gfx_v9_4.h"
  52 #include "gfx_v9_0.h"
  53 #include "gfx_v9_4_2.h"
  54
  55 #include "asic_reg/pwr/pwr_10_0_offset.h"
  56 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
  57 #include "asic_reg/gc/gc_9_0_default.h"
  58
  59 #define GFX9_NUM_GFX_RINGS     1
  60 #define GFX9_NUM_SW_GFX_RINGS  2
  61 #define GFX9_MEC_HPD_SIZE 4096
  62 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
  63 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
  64
  65 #define mmGCEA_PROBE_MAP                        0x070c
  66 #define mmGCEA_PROBE_MAP_BASE_IDX               0
  67
  68 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  69 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  70 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  71 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  72 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  73 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  74
  75 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
  76 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
  77 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
  78 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
  79 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
  80 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
  81
  82 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
  83 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
  84 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
  85 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
  86 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
  87 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
  88
  89 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
  91 MODULE_FIRMWARE("amdgpu/raven_me.bin");
  92 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
  93 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
  94 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
  95
  96 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
  97 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
  98 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
  99 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
 100 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
 101 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
 102 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
 103
 104 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
 105 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
 106 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
 107 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
 108 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
 109 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
 110 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
 111
 112 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
 113 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
 114
 115 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
 116 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
 117 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
 118 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
 119 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
 120
 121 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
 122 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
 123 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
 124 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
 125 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
 126 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
 127
 128 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
 129 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
 130 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
 131 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");
 132 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
 133
 134 #define mmTCP_CHAN_STEER_0_ARCT                                                         0x0b03
 135 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX                                                        0
 136 #define mmTCP_CHAN_STEER_1_ARCT                                                         0x0b04
 137 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX                                                        0
 138 #define mmTCP_CHAN_STEER_2_ARCT                                                         0x0b09
 139 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX                                                        0
 140 #define mmTCP_CHAN_STEER_3_ARCT                                                         0x0b0a
 141 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX                                                        0
 142 #define mmTCP_CHAN_STEER_4_ARCT                                                         0x0b0b
 143 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX                                                        0
 144 #define mmTCP_CHAN_STEER_5_ARCT                                                         0x0b0c
 145 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX                                                        0
 146
 147 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir                0x0025
 148 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX       1
 149 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir                0x0026
 150 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX       1
 151
 152 enum ta_ras_gfx_subblock {
 153         /*CPC*/
 154         TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
 155         TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
 156         TA_RAS_BLOCK__GFX_CPC_UCODE,
 157         TA_RAS_BLOCK__GFX_DC_STATE_ME1,
 158         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
 159         TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
 160         TA_RAS_BLOCK__GFX_DC_STATE_ME2,
 161         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
 162         TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 163         TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 164         /* CPF*/
 165         TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 166         TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 167         TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
 168         TA_RAS_BLOCK__GFX_CPF_TAG,
 169         TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
 170         /* CPG*/
 171         TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 172         TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 173         TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
 174         TA_RAS_BLOCK__GFX_CPG_TAG,
 175         TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
 176         /* GDS*/
 177         TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 178         TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 179         TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
 180         TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
 181         TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
 182         TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 183         TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 184         /* SPI*/
 185         TA_RAS_BLOCK__GFX_SPI_SR_MEM,
 186         /* SQ*/
 187         TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 188         TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 189         TA_RAS_BLOCK__GFX_SQ_LDS_D,
 190         TA_RAS_BLOCK__GFX_SQ_LDS_I,
 191         TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
 192         TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
 193         /* SQC (3 ranges)*/
 194         TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 195         /* SQC range 0*/
 196         TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 197         TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
 198                 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
 199         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
 200         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
 201         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
 202         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
 203         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
 204         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 205         TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
 206                 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 207         /* SQC range 1*/
 208         TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 209         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
 210                 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 211         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
 212         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
 213         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
 214         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
 215         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
 216         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
 217         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
 218         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 219         TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
 220                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 221         /* SQC range 2*/
 222         TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 223         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
 224                 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 225         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
 226         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
 227         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
 228         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
 229         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
 230         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
 231         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
 232         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 233         TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
 234                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 235         TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
 236         /* TA*/
 237         TA_RAS_BLOCK__GFX_TA_INDEX_START,
 238         TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
 239         TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
 240         TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
 241         TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
 242         TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 243         TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 244         /* TCA*/
 245         TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 246         TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 247         TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 248         TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 249         /* TCC (5 sub-ranges)*/
 250         TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 251         /* TCC range 0*/
 252         TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 253         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
 254         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
 255         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
 256         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
 257         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
 258         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
 259         TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
 260         TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 261         TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 262         /* TCC range 1*/
 263         TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 264         TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 265         TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 266         TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
 267                 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 268         /* TCC range 2*/
 269         TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 270         TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 271         TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
 272         TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
 273         TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
 274         TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
 275         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
 276         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
 277         TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 278         TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
 279                 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 280         /* TCC range 3*/
 281         TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 282         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 283         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 284         TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
 285                 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 286         /* TCC range 4*/
 287         TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 288         TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
 289                 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 290         TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 291         TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
 292                 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 293         TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
 294         /* TCI*/
 295         TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
 296         /* TCP*/
 297         TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 298         TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 299         TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
 300         TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
 301         TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
 302         TA_RAS_BLOCK__GFX_TCP_DB_RAM,
 303         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
 304         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 305         TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 306         /* TD*/
 307         TA_RAS_BLOCK__GFX_TD_INDEX_START,
 308         TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
 309         TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
 310         TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 311         TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 312         /* EA (3 sub-ranges)*/
 313         TA_RAS_BLOCK__GFX_EA_INDEX_START,
 314         /* EA range 0*/
 315         TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
 316         TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
 317         TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
 318         TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
 319         TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
 320         TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
 321         TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
 322         TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
 323         TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 324         TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 325         /* EA range 1*/
 326         TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 327         TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 328         TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
 329         TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
 330         TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
 331         TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
 332         TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
 333         TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 334         TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 335         /* EA range 2*/
 336         TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 337         TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 338         TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
 339         TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
 340         TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 341         TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 342         TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
 343         /* UTC VM L2 bank*/
 344         TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
 345         /* UTC VM walker*/
 346         TA_RAS_BLOCK__UTC_VML2_WALKER,
 347         /* UTC ATC L2 2MB cache*/
 348         TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
 349         /* UTC ATC L2 4KB cache*/
 350         TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
 351         TA_RAS_BLOCK__GFX_MAX
 352 };
 353
 354 struct ras_gfx_subblock {
 355         unsigned char *name;
 356         int ta_subblock;
 357         int hw_supported_error_type;
 358         int sw_supported_error_type;
 359 };
 360
 361 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \
 362         [AMDGPU_RAS_BLOCK__##subblock] = {                                     \
 363                 #subblock,                                                     \
 364                 TA_RAS_BLOCK__##subblock,                                      \
 365                 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \
 366                 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \
 367         }
 368
 369 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
 370         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
 371         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
 372         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 373         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 374         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 375         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 376         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 377         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 378         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 379         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 380         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
 381         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
 382         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
 383         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
 384         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 385         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
 386         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
 387                              0),
 388         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
 389                              0),
 390         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 391         AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 392         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 393         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
 394         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
 395         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 396         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
 397         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 398                              0, 0),
 399         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 400                              0),
 401         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 402                              0, 0),
 403         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
 404                              0),
 405         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 406                              0, 0),
 407         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 408                              0),
 409         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 410                              1),
 411         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 412                              0, 0, 0),
 413         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 414                              0),
 415         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 416                              0),
 417         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 418                              0),
 419         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 420                              0),
 421         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 422                              0),
 423         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 424                              0, 0),
 425         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 426                              0),
 427         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 428                              0),
 429         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 430                              0, 0, 0),
 431         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 432                              0),
 433         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 434                              0),
 435         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 436                              0),
 437         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 438                              0),
 439         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 440                              0),
 441         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 442                              0, 0),
 443         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 444                              0),
 445         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
 446         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 447         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 448         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 449         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 450         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
 451         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 452         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
 453         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
 454                              1),
 455         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
 456                              1),
 457         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
 458                              1),
 459         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
 460                              0),
 461         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
 462                              0),
 463         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 464         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 465         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
 466         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
 467         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
 468         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
 469         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 470         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
 471         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
 472         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 473         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
 474         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
 475                              0),
 476         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 477         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
 478                              0),
 479         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
 480                              0, 0),
 481         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
 482                              0),
 483         AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 484         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
 485         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
 486         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 487         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 488         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 489         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
 490         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
 491         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
 492         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
 493         AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 494         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
 495         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 496         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 497         AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 498         AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 499         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 500         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 501         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 502         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 503         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 504         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 505         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 506         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 507         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 508         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 509         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 510         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 511         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 512         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 513         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
 514         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
 515         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
 516         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
 517 };
 518
 519 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
 520 {
 521         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 522         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
 523         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 524         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 525         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 526         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 527         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 528         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 529         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 530         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
 531         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
 532         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 533         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 534         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 535         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 536         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 537         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
 538         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 539         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 540         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 541 };
 542
 543 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 544 {
 545         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 546         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 547         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 548         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 549         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 550         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 551         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
 552         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 553         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
 554         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 555         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 556         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 557         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 558         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 559         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 560         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
 561         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
 562         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 563 };
 564
 565 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
 566 {
 567         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
 568         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 569         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 570         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
 571         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
 572         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
 573         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
 574         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
 575         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
 576         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
 577         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
 578 };
 579
 580 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 581 {
 582         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 583         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 584         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 585         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 586         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 587         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 588         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 589         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 590         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 591         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 592         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 593         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 594         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 595         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 596         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 597         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 598         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 599         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 600         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 601         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
 602         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
 603         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 604         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 605         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 606 };
 607
 608 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 609 {
 610         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 611         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
 612         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
 613         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
 614         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
 615         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 616         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 617 };
 618
 619 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
 620 {
 621         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
 622         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 623         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 624         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
 625         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
 626         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
 627         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
 628         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
 629         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 630         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 631         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
 632         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
 633         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
 634         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 635         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 636         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 637         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
 638         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 639         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 640 };
 641
 642 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
 643 {
 644         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 645         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 646         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 647         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
 648         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
 649         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 650         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 651         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 652         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 653         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 654         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 655         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
 656 };
 657
 658 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 659 {
 660         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
 661         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
 662         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 663 };
 664
 665 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
 666 {
 667         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 668         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 669         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 670         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 671         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 672         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 673         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 674         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 675         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
 676         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 677         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 678         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 679         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 680         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 681         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 682         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 683 };
 684
 685 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 686 {
 687         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
 688         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 689         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 690         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
 691         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
 692         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 693         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
 694         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 695         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
 696         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 697         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 698         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 699         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 700 };
 701
 702 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
 703 {
 704         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 705         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
 706         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
 707         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
 708         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
 709         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
 710         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
 711         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
 712         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
 713         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
 714         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
 715 };
 716
 717 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
 718         {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
 719         {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
 720 };
 721
 722 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
 723 {
 724         mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 725         mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 726         mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 727         mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 728         mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 729         mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 730         mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 731         mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 732 };
 733
 734 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 735 {
 736         mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 737         mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 738         mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 739         mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 740         mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 741         mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 742         mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 743         mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 744 };
 745
 746 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 747 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 748 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 749 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
 750
 751 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 752 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 753 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 754 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 755 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 756                                 struct amdgpu_cu_info *cu_info);
 757 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 758 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
 759 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
 760 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
 761                                           void *ras_error_status);
 762 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
 763                                      void *inject_if);
 764 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
 765
 766 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 767                                 uint64_t queue_mask)
 768 {
 769         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 770         amdgpu_ring_write(kiq_ring,
 771                 PACKET3_SET_RESOURCES_VMID_MASK(0) |
 772                 /* vmid_mask:0* queue_type:0 (KIQ) */
 773                 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
 774         amdgpu_ring_write(kiq_ring,
 775                         lower_32_bits(queue_mask));     /* queue mask lo */
 776         amdgpu_ring_write(kiq_ring,
 777                         upper_32_bits(queue_mask));     /* queue mask hi */
 778         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 779         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 780         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 781         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 782 }
 783
 784 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 785                                  struct amdgpu_ring *ring)
 786 {
 787         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 788         uint64_t wptr_addr = ring->wptr_gpu_addr;
 789         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 790
 791         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 792         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 793         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 794                          PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 795                          PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 796                          PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 797                          PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 798                          PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
 799                          /*queue_type: normal compute queue */
 800                          PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
 801                          /* alloc format: all_on_one_pipe */
 802                          PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
 803                          PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 804                          /* num_queues: must be 1 */
 805                          PACKET3_MAP_QUEUES_NUM_QUEUES(1));
 806         amdgpu_ring_write(kiq_ring,
 807                         PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 808         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 809         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 810         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 811         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 812 }
 813
 814 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 815                                    struct amdgpu_ring *ring,
 816                                    enum amdgpu_unmap_queues_action action,
 817                                    u64 gpu_addr, u64 seq)
 818 {
 819         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 820
 821         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 822         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 823                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 824                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 825                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 826                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 827         amdgpu_ring_write(kiq_ring,
 828                         PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 829
 830         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 831                 amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));
 832                 amdgpu_ring_write(kiq_ring, 0);
 833                 amdgpu_ring_write(kiq_ring, 0);
 834
 835         } else {
 836                 amdgpu_ring_write(kiq_ring, 0);
 837                 amdgpu_ring_write(kiq_ring, 0);
 838                 amdgpu_ring_write(kiq_ring, 0);
 839         }
 840 }
 841
 842 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 843                                    struct amdgpu_ring *ring,
 844                                    u64 addr,
 845                                    u64 seq)
 846 {
 847         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 848
 849         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 850         amdgpu_ring_write(kiq_ring,
 851                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 852                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 853                           PACKET3_QUERY_STATUS_COMMAND(2));
 854         /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 855         amdgpu_ring_write(kiq_ring,
 856                         PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 857                         PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 858         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 859         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 860         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 861         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 862 }
 863
 864 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 865                                 uint16_t pasid, uint32_t flush_type,
 866                                 bool all_hub)
 867 {
 868         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
 869         amdgpu_ring_write(kiq_ring,
 870                         PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
 871                         PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
 872                         PACKET3_INVALIDATE_TLBS_PASID(pasid) |
 873                         PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
 874 }
 875
 876 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
 877         .kiq_set_resources = gfx_v9_0_kiq_set_resources,
 878         .kiq_map_queues = gfx_v9_0_kiq_map_queues,
 879         .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
 880         .kiq_query_status = gfx_v9_0_kiq_query_status,
 881         .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
 882         .set_resources_size = 8,
 883         .map_queues_size = 7,
 884         .unmap_queues_size = 6,
 885         .query_status_size = 7,
 886         .invalidate_tlbs_size = 2,
 887 };
 888
 889 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 890 {
 891         adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
 892 }
 893
 894 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 895 {
 896         switch (adev->ip_versions[GC_HWIP][0]) {
 897         case IP_VERSION(9, 0, 1):
 898                 soc15_program_register_sequence(adev,
 899                                                 golden_settings_gc_9_0,
 900                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 901                 soc15_program_register_sequence(adev,
 902                                                 golden_settings_gc_9_0_vg10,
 903                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
 904                 break;
 905         case IP_VERSION(9, 2, 1):
 906                 soc15_program_register_sequence(adev,
 907                                                 golden_settings_gc_9_2_1,
 908                                                 ARRAY_SIZE(golden_settings_gc_9_2_1));
 909                 soc15_program_register_sequence(adev,
 910                                                 golden_settings_gc_9_2_1_vg12,
 911                                                 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
 912                 break;
 913         case IP_VERSION(9, 4, 0):
 914                 soc15_program_register_sequence(adev,
 915                                                 golden_settings_gc_9_0,
 916                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 917                 soc15_program_register_sequence(adev,
 918                                                 golden_settings_gc_9_0_vg20,
 919                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
 920                 break;
 921         case IP_VERSION(9, 4, 1):
 922                 soc15_program_register_sequence(adev,
 923                                                 golden_settings_gc_9_4_1_arct,
 924                                                 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
 925                 break;
 926         case IP_VERSION(9, 2, 2):
 927         case IP_VERSION(9, 1, 0):
 928                 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
 929                                                 ARRAY_SIZE(golden_settings_gc_9_1));
 930                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
 931                         soc15_program_register_sequence(adev,
 932                                                         golden_settings_gc_9_1_rv2,
 933                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv2));
 934                 else
 935                         soc15_program_register_sequence(adev,
 936                                                         golden_settings_gc_9_1_rv1,
 937                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv1));
 938                 break;
 939          case IP_VERSION(9, 3, 0):
 940                 soc15_program_register_sequence(adev,
 941                                                 golden_settings_gc_9_1_rn,
 942                                                 ARRAY_SIZE(golden_settings_gc_9_1_rn));
 943                 return; /* for renoir, don't need common goldensetting */
 944         case IP_VERSION(9, 4, 2):
 945                 gfx_v9_4_2_init_golden_registers(adev,
 946                                                  adev->smuio.funcs->get_die_id(adev));
 947                 break;
 948         default:
 949                 break;
 950         }
 951
 952         if ((adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) &&
 953             (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 2)))
 954                 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
 955                                                 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
 956 }
 957
 958 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
 959                                        bool wc, uint32_t reg, uint32_t val)
 960 {
 961         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 962         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
 963                                 WRITE_DATA_DST_SEL(0) |
 964                                 (wc ? WR_CONFIRM : 0));
 965         amdgpu_ring_write(ring, reg);
 966         amdgpu_ring_write(ring, 0);
 967         amdgpu_ring_write(ring, val);
 968 }
 969
 970 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 971                                   int mem_space, int opt, uint32_t addr0,
 972                                   uint32_t addr1, uint32_t ref, uint32_t mask,
 973                                   uint32_t inv)
 974 {
 975         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 976         amdgpu_ring_write(ring,
 977                                  /* memory (1) or register (0) */
 978                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 979                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
 980                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
 981                                  WAIT_REG_MEM_ENGINE(eng_sel)));
 982
 983         if (mem_space)
 984                 BUG_ON(addr0 & 0x3); /* Dword align */
 985         amdgpu_ring_write(ring, addr0);
 986         amdgpu_ring_write(ring, addr1);
 987         amdgpu_ring_write(ring, ref);
 988         amdgpu_ring_write(ring, mask);
 989         amdgpu_ring_write(ring, inv); /* poll interval */
 990 }
 991
 992 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
 993 {
 994         struct amdgpu_device *adev = ring->adev;
 995         uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
 996         uint32_t tmp = 0;
 997         unsigned i;
 998         int r;
 999
1000         WREG32(scratch, 0xCAFEDEAD);
1001         r = amdgpu_ring_alloc(ring, 3);
1002         if (r)
1003                 return r;
1004
1005         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1006         amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START);
1007         amdgpu_ring_write(ring, 0xDEADBEEF);
1008         amdgpu_ring_commit(ring);
1009
1010         for (i = 0; i < adev->usec_timeout; i++) {
1011                 tmp = RREG32(scratch);
1012                 if (tmp == 0xDEADBEEF)
1013                         break;
1014                 udelay(1);
1015         }
1016
1017         if (i >= adev->usec_timeout)
1018                 r = -ETIMEDOUT;
1019         return r;
1020 }
1021
1022 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1023 {
1024         struct amdgpu_device *adev = ring->adev;
1025         struct amdgpu_ib ib;
1026         struct dma_fence *f = NULL;
1027
1028         unsigned index;
1029         uint64_t gpu_addr;
1030         uint32_t tmp;
1031         long r;
1032
1033         r = amdgpu_device_wb_get(adev, &index);
1034         if (r)
1035                 return r;
1036
1037         gpu_addr = adev->wb.gpu_addr + (index * 4);
1038         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1039         memset(&ib, 0, sizeof(ib));
1040         r = amdgpu_ib_get(adev, NULL, 16,
1041                                         AMDGPU_IB_POOL_DIRECT, &ib);
1042         if (r)
1043                 goto err1;
1044
1045         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1046         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1047         ib.ptr[2] = lower_32_bits(gpu_addr);
1048         ib.ptr[3] = upper_32_bits(gpu_addr);
1049         ib.ptr[4] = 0xDEADBEEF;
1050         ib.length_dw = 5;
1051
1052         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1053         if (r)
1054                 goto err2;
1055
1056         r = dma_fence_wait_timeout(f, false, timeout);
1057         if (r == 0) {
1058                 r = -ETIMEDOUT;
1059                 goto err2;
1060         } else if (r < 0) {
1061                 goto err2;
1062         }
1063
1064         tmp = adev->wb.wb[index];
1065         if (tmp == 0xDEADBEEF)
1066                 r = 0;
1067         else
1068                 r = -EINVAL;
1069
1070 err2:
1071         amdgpu_ib_free(adev, &ib, NULL);
1072         dma_fence_put(f);
1073 err1:
1074         amdgpu_device_wb_free(adev, index);
1075         return r;
1076 }
1077
1078
1079 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1080 {
1081         amdgpu_ucode_release(&adev->gfx.pfp_fw);
1082         amdgpu_ucode_release(&adev->gfx.me_fw);
1083         amdgpu_ucode_release(&adev->gfx.ce_fw);
1084         amdgpu_ucode_release(&adev->gfx.rlc_fw);
1085         amdgpu_ucode_release(&adev->gfx.mec_fw);
1086         amdgpu_ucode_release(&adev->gfx.mec2_fw);
1087
1088         kfree(adev->gfx.rlc.register_list_format);
1089 }
1090
1091 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1092 {
1093         adev->gfx.me_fw_write_wait = false;
1094         adev->gfx.mec_fw_write_wait = false;
1095
1096         if ((adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) &&
1097             ((adev->gfx.mec_fw_version < 0x000001a5) ||
1098             (adev->gfx.mec_feature_version < 46) ||
1099             (adev->gfx.pfp_fw_version < 0x000000b7) ||
1100             (adev->gfx.pfp_feature_version < 46)))
1101                 DRM_WARN_ONCE("CP firmware version too old, please update!");
1102
1103         switch (adev->ip_versions[GC_HWIP][0]) {
1104         case IP_VERSION(9, 0, 1):
1105                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1106                     (adev->gfx.me_feature_version >= 42) &&
1107                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1108                     (adev->gfx.pfp_feature_version >= 42))
1109                         adev->gfx.me_fw_write_wait = true;
1110
1111                 if ((adev->gfx.mec_fw_version >=  0x00000193) &&
1112                     (adev->gfx.mec_feature_version >= 42))
1113                         adev->gfx.mec_fw_write_wait = true;
1114                 break;
1115         case IP_VERSION(9, 2, 1):
1116                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1117                     (adev->gfx.me_feature_version >= 44) &&
1118                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1119                     (adev->gfx.pfp_feature_version >= 44))
1120                         adev->gfx.me_fw_write_wait = true;
1121
1122                 if ((adev->gfx.mec_fw_version >=  0x00000196) &&
1123                     (adev->gfx.mec_feature_version >= 44))
1124                         adev->gfx.mec_fw_write_wait = true;
1125                 break;
1126         case IP_VERSION(9, 4, 0):
1127                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1128                     (adev->gfx.me_feature_version >= 44) &&
1129                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1130                     (adev->gfx.pfp_feature_version >= 44))
1131                         adev->gfx.me_fw_write_wait = true;
1132
1133                 if ((adev->gfx.mec_fw_version >=  0x00000197) &&
1134                     (adev->gfx.mec_feature_version >= 44))
1135                         adev->gfx.mec_fw_write_wait = true;
1136                 break;
1137         case IP_VERSION(9, 1, 0):
1138         case IP_VERSION(9, 2, 2):
1139                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1140                     (adev->gfx.me_feature_version >= 42) &&
1141                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1142                     (adev->gfx.pfp_feature_version >= 42))
1143                         adev->gfx.me_fw_write_wait = true;
1144
1145                 if ((adev->gfx.mec_fw_version >=  0x00000192) &&
1146                     (adev->gfx.mec_feature_version >= 42))
1147                         adev->gfx.mec_fw_write_wait = true;
1148                 break;
1149         default:
1150                 adev->gfx.me_fw_write_wait = true;
1151                 adev->gfx.mec_fw_write_wait = true;
1152                 break;
1153         }
1154 }
1155
1156 struct amdgpu_gfxoff_quirk {
1157         u16 chip_vendor;
1158         u16 chip_device;
1159         u16 subsys_vendor;
1160         u16 subsys_device;
1161         u8 revision;
1162 };
1163
1164 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1165         /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1166         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1167         /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1168         { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1169         /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1170         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1171         /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1172         { 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
1173         { 0, 0, 0, 0, 0 },
1174 };
1175
1176 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1177 {
1178         const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1179
1180         while (p && p->chip_device != 0) {
1181                 if (pdev->vendor == p->chip_vendor &&
1182                     pdev->device == p->chip_device &&
1183                     pdev->subsystem_vendor == p->subsys_vendor &&
1184                     pdev->subsystem_device == p->subsys_device &&
1185                     pdev->revision == p->revision) {
1186                         return true;
1187                 }
1188                 ++p;
1189         }
1190         return false;
1191 }
1192
1193 static bool is_raven_kicker(struct amdgpu_device *adev)
1194 {
1195         if (adev->pm.fw_version >= 0x41e2b)
1196                 return true;
1197         else
1198                 return false;
1199 }
1200
1201 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1202 {
1203         if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 3, 0)) &&
1204             (adev->gfx.me_fw_version >= 0x000000a5) &&
1205             (adev->gfx.me_feature_version >= 52))
1206                 return true;
1207         else
1208                 return false;
1209 }
1210
1211 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1212 {
1213         if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1214                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1215
1216         switch (adev->ip_versions[GC_HWIP][0]) {
1217         case IP_VERSION(9, 0, 1):
1218         case IP_VERSION(9, 2, 1):
1219         case IP_VERSION(9, 4, 0):
1220                 break;
1221         case IP_VERSION(9, 2, 2):
1222         case IP_VERSION(9, 1, 0):
1223                 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1224                       (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1225                     ((!is_raven_kicker(adev) &&
1226                       adev->gfx.rlc_fw_version < 531) ||
1227                      (adev->gfx.rlc_feature_version < 1) ||
1228                      !adev->gfx.rlc.is_rlc_v2_1))
1229                         adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1230
1231                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1232                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1233                                 AMD_PG_SUPPORT_CP |
1234                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1235                 break;
1236         case IP_VERSION(9, 3, 0):
1237                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1238                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1239                                 AMD_PG_SUPPORT_CP |
1240                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1241                 break;
1242         default:
1243                 break;
1244         }
1245 }
1246
1247 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1248                                           char *chip_name)
1249 {
1250         char fw_name[30];
1251         int err;
1252
1253         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1254         err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, fw_name);
1255         if (err)
1256                 goto out;
1257         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
1258
1259         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1260         err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, fw_name);
1261         if (err)
1262                 goto out;
1263         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
1264
1265         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1266         err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, fw_name);
1267         if (err)
1268                 goto out;
1269         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE);
1270
1271 out:
1272         if (err) {
1273                 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1274                 amdgpu_ucode_release(&adev->gfx.me_fw);
1275                 amdgpu_ucode_release(&adev->gfx.ce_fw);
1276         }
1277         return err;
1278 }
1279
1280 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1281                                        char *chip_name)
1282 {
1283         char fw_name[30];
1284         int err;
1285         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1286         uint16_t version_major;
1287         uint16_t version_minor;
1288         uint32_t smu_version;
1289
1290         /*
1291          * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1292          * instead of picasso_rlc.bin.
1293          * Judgment method:
1294          * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1295          *          or revision >= 0xD8 && revision <= 0xDF
1296          * otherwise is PCO FP5
1297          */
1298         if (!strcmp(chip_name, "picasso") &&
1299                 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1300                 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1301                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1302         else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1303                 (smu_version >= 0x41e2b))
1304                 /**
1305                 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1306                 */
1307                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1308         else
1309                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1310         err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, fw_name);
1311         if (err)
1312                 goto out;
1313         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1314
1315         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1316         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1317         err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
1318 out:
1319         if (err)
1320                 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1321
1322         return err;
1323 }
1324
1325 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1326 {
1327         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
1328             adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) ||
1329             adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 3, 0))
1330                 return false;
1331
1332         return true;
1333 }
1334
1335 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1336                                               char *chip_name)
1337 {
1338         char fw_name[30];
1339         int err;
1340
1341         if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1342                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sjt_mec.bin", chip_name);
1343         else
1344                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1345
1346         err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name);
1347         if (err)
1348                 goto out;
1349         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
1350         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
1351
1352         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1353                 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1354                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sjt_mec2.bin", chip_name);
1355                 else
1356                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1357
1358                 /* ignore failures to load */
1359                 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, fw_name);
1360                 if (!err) {
1361                         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2);
1362                         amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT);
1363                 } else {
1364                         err = 0;
1365                         amdgpu_ucode_release(&adev->gfx.mec2_fw);
1366                 }
1367         } else {
1368                 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1369                 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1370         }
1371
1372         gfx_v9_0_check_if_need_gfxoff(adev);
1373         gfx_v9_0_check_fw_write_wait(adev);
1374
1375 out:
1376         if (err)
1377                 amdgpu_ucode_release(&adev->gfx.mec_fw);
1378         return err;
1379 }
1380
1381 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1382 {
1383         char ucode_prefix[30];
1384         int r;
1385
1386         DRM_DEBUG("\n");
1387         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
1388
1389         /* No CPG in Arcturus */
1390         if (adev->gfx.num_gfx_rings) {
1391                 r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix);
1392                 if (r)
1393                         return r;
1394         }
1395
1396         r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix);
1397         if (r)
1398                 return r;
1399
1400         r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix);
1401         if (r)
1402                 return r;
1403
1404         return r;
1405 }
1406
1407 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1408 {
1409         u32 count = 0;
1410         const struct cs_section_def *sect = NULL;
1411         const struct cs_extent_def *ext = NULL;
1412
1413         /* begin clear state */
1414         count += 2;
1415         /* context control state */
1416         count += 3;
1417
1418         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1419                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1420                         if (sect->id == SECT_CONTEXT)
1421                                 count += 2 + ext->reg_count;
1422                         else
1423                                 return 0;
1424                 }
1425         }
1426
1427         /* end clear state */
1428         count += 2;
1429         /* clear state */
1430         count += 2;
1431
1432         return count;
1433 }
1434
1435 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1436                                     volatile u32 *buffer)
1437 {
1438         u32 count = 0, i;
1439         const struct cs_section_def *sect = NULL;
1440         const struct cs_extent_def *ext = NULL;
1441
1442         if (adev->gfx.rlc.cs_data == NULL)
1443                 return;
1444         if (buffer == NULL)
1445                 return;
1446
1447         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1448         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1449
1450         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1451         buffer[count++] = cpu_to_le32(0x80000000);
1452         buffer[count++] = cpu_to_le32(0x80000000);
1453
1454         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1455                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1456                         if (sect->id == SECT_CONTEXT) {
1457                                 buffer[count++] =
1458                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1459                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1460                                                 PACKET3_SET_CONTEXT_REG_START);
1461                                 for (i = 0; i < ext->reg_count; i++)
1462                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1463                         } else {
1464                                 return;
1465                         }
1466                 }
1467         }
1468
1469         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1470         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1471
1472         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1473         buffer[count++] = cpu_to_le32(0);
1474 }
1475
1476 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1477 {
1478         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1479         uint32_t pg_always_on_cu_num = 2;
1480         uint32_t always_on_cu_num;
1481         uint32_t i, j, k;
1482         uint32_t mask, cu_bitmap, counter;
1483
1484         if (adev->flags & AMD_IS_APU)
1485                 always_on_cu_num = 4;
1486         else if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 2, 1))
1487                 always_on_cu_num = 8;
1488         else
1489                 always_on_cu_num = 12;
1490
1491         mutex_lock(&adev->grbm_idx_mutex);
1492         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1493                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1494                         mask = 1;
1495                         cu_bitmap = 0;
1496                         counter = 0;
1497                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff);
1498
1499                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1500                                 if (cu_info->bitmap[i][j] & mask) {
1501                                         if (counter == pg_always_on_cu_num)
1502                                                 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1503                                         if (counter < always_on_cu_num)
1504                                                 cu_bitmap |= mask;
1505                                         else
1506                                                 break;
1507                                         counter++;
1508                                 }
1509                                 mask <<= 1;
1510                         }
1511
1512                         WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1513                         cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1514                 }
1515         }
1516         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1517         mutex_unlock(&adev->grbm_idx_mutex);
1518 }
1519
1520 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1521 {
1522         uint32_t data;
1523
1524         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1525         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1526         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1527         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1528         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1529
1530         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1531         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1532
1533         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1534         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1535
1536         mutex_lock(&adev->grbm_idx_mutex);
1537         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1538         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1539         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1540
1541         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1542         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1543         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1544         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1545         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1546
1547         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1548         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1549         data &= 0x0000FFFF;
1550         data |= 0x00C00000;
1551         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1552
1553         /*
1554          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1555          * programmed in gfx_v9_0_init_always_on_cu_mask()
1556          */
1557
1558         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1559          * but used for RLC_LB_CNTL configuration */
1560         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1561         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1562         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1563         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1564         mutex_unlock(&adev->grbm_idx_mutex);
1565
1566         gfx_v9_0_init_always_on_cu_mask(adev);
1567 }
1568
1569 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1570 {
1571         uint32_t data;
1572
1573         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1574         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1575         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1576         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1577         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1578
1579         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1580         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1581
1582         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1583         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1584
1585         mutex_lock(&adev->grbm_idx_mutex);
1586         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1587         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1588         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1589
1590         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1591         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1592         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1593         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1594         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1595
1596         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1597         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1598         data &= 0x0000FFFF;
1599         data |= 0x00C00000;
1600         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1601
1602         /*
1603          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1604          * programmed in gfx_v9_0_init_always_on_cu_mask()
1605          */
1606
1607         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1608          * but used for RLC_LB_CNTL configuration */
1609         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1610         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1611         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1612         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1613         mutex_unlock(&adev->grbm_idx_mutex);
1614
1615         gfx_v9_0_init_always_on_cu_mask(adev);
1616 }
1617
1618 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1619 {
1620         WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1621 }
1622
1623 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1624 {
1625         if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1626                 return 5;
1627         else
1628                 return 4;
1629 }
1630
1631 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1632 {
1633         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1634
1635         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl;
1636         reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1637         reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);
1638         reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);
1639         reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);
1640         reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);
1641         reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);
1642         reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);
1643         adev->gfx.rlc.rlcg_reg_access_supported = true;
1644 }
1645
1646 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1647 {
1648         const struct cs_section_def *cs_data;
1649         int r;
1650
1651         adev->gfx.rlc.cs_data = gfx9_cs_data;
1652
1653         cs_data = adev->gfx.rlc.cs_data;
1654
1655         if (cs_data) {
1656                 /* init clear state block */
1657                 r = amdgpu_gfx_rlc_init_csb(adev);
1658                 if (r)
1659                         return r;
1660         }
1661
1662         if (adev->flags & AMD_IS_APU) {
1663                 /* TODO: double check the cp_table_size for RV */
1664                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1665                 r = amdgpu_gfx_rlc_init_cpt(adev);
1666                 if (r)
1667                         return r;
1668         }
1669
1670         switch (adev->ip_versions[GC_HWIP][0]) {
1671         case IP_VERSION(9, 2, 2):
1672         case IP_VERSION(9, 1, 0):
1673                 gfx_v9_0_init_lbpw(adev);
1674                 break;
1675         case IP_VERSION(9, 4, 0):
1676                 gfx_v9_4_init_lbpw(adev);
1677                 break;
1678         default:
1679                 break;
1680         }
1681
1682         /* init spm vmid with 0xf */
1683         if (adev->gfx.rlc.funcs->update_spm_vmid)
1684                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1685
1686         return 0;
1687 }
1688
1689 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1690 {
1691         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1692         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1693 }
1694
1695 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1696 {
1697         int r;
1698         u32 *hpd;
1699         const __le32 *fw_data;
1700         unsigned fw_size;
1701         u32 *fw;
1702         size_t mec_hpd_size;
1703
1704         const struct gfx_firmware_header_v1_0 *mec_hdr;
1705
1706         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1707
1708         /* take ownership of the relevant compute queues */
1709         amdgpu_gfx_compute_queue_acquire(adev);
1710         mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1711         if (mec_hpd_size) {
1712                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1713                                               AMDGPU_GEM_DOMAIN_VRAM |
1714                                               AMDGPU_GEM_DOMAIN_GTT,
1715                                               &adev->gfx.mec.hpd_eop_obj,
1716                                               &adev->gfx.mec.hpd_eop_gpu_addr,
1717                                               (void **)&hpd);
1718                 if (r) {
1719                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1720                         gfx_v9_0_mec_fini(adev);
1721                         return r;
1722                 }
1723
1724                 memset(hpd, 0, mec_hpd_size);
1725
1726                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1727                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1728         }
1729
1730         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1731
1732         fw_data = (const __le32 *)
1733                 (adev->gfx.mec_fw->data +
1734                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1735         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1736
1737         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1738                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1739                                       &adev->gfx.mec.mec_fw_obj,
1740                                       &adev->gfx.mec.mec_fw_gpu_addr,
1741                                       (void **)&fw);
1742         if (r) {
1743                 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1744                 gfx_v9_0_mec_fini(adev);
1745                 return r;
1746         }
1747
1748         memcpy(fw, fw_data, fw_size);
1749
1750         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1751         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1752
1753         return 0;
1754 }
1755
1756 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1757 {
1758         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1759                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1760                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1761                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
1762                 (SQ_IND_INDEX__FORCE_READ_MASK));
1763         return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1764 }
1765
1766 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1767                            uint32_t wave, uint32_t thread,
1768                            uint32_t regno, uint32_t num, uint32_t *out)
1769 {
1770         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1771                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1772                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1773                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1774                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1775                 (SQ_IND_INDEX__FORCE_READ_MASK) |
1776                 (SQ_IND_INDEX__AUTO_INCR_MASK));
1777         while (num--)
1778                 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1779 }
1780
1781 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1782 {
1783         /* type 1 wave data */
1784         dst[(*no_fields)++] = 1;
1785         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1786         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1787         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1788         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1789         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1790         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1791         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1792         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1793         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1794         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1795         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1796         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1797         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1798         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1799         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
1800 }
1801
1802 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
1803                                      uint32_t wave, uint32_t start,
1804                                      uint32_t size, uint32_t *dst)
1805 {
1806         wave_read_regs(
1807                 adev, simd, wave, 0,
1808                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
1809 }
1810
1811 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
1812                                      uint32_t wave, uint32_t thread,
1813                                      uint32_t start, uint32_t size,
1814                                      uint32_t *dst)
1815 {
1816         wave_read_regs(
1817                 adev, simd, wave, thread,
1818                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1819 }
1820
1821 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
1822                                   u32 me, u32 pipe, u32 q, u32 vm)
1823 {
1824         soc15_grbm_select(adev, me, pipe, q, vm);
1825 }
1826
1827 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
1828         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
1829         .select_se_sh = &gfx_v9_0_select_se_sh,
1830         .read_wave_data = &gfx_v9_0_read_wave_data,
1831         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
1832         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
1833         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
1834 };
1835
1836 const struct amdgpu_ras_block_hw_ops  gfx_v9_0_ras_ops = {
1837                 .ras_error_inject = &gfx_v9_0_ras_error_inject,
1838                 .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
1839                 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
1840 };
1841
1842 static struct amdgpu_gfx_ras gfx_v9_0_ras = {
1843         .ras_block = {
1844                 .hw_ops = &gfx_v9_0_ras_ops,
1845         },
1846 };
1847
1848 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
1849 {
1850         u32 gb_addr_config;
1851         int err;
1852
1853         switch (adev->ip_versions[GC_HWIP][0]) {
1854         case IP_VERSION(9, 0, 1):
1855                 adev->gfx.config.max_hw_contexts = 8;
1856                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1857                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1858                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1859                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1860                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
1861                 break;
1862         case IP_VERSION(9, 2, 1):
1863                 adev->gfx.config.max_hw_contexts = 8;
1864                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1865                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1866                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1867                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1868                 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
1869                 DRM_INFO("fix gfx.config for vega12\n");
1870                 break;
1871         case IP_VERSION(9, 4, 0):
1872                 adev->gfx.ras = &gfx_v9_0_ras;
1873                 adev->gfx.config.max_hw_contexts = 8;
1874                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1875                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1876                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1877                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1878                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1879                 gb_addr_config &= ~0xf3e777ff;
1880                 gb_addr_config |= 0x22014042;
1881                 /* check vbios table if gpu info is not available */
1882                 err = amdgpu_atomfirmware_get_gfx_info(adev);
1883                 if (err)
1884                         return err;
1885                 break;
1886         case IP_VERSION(9, 2, 2):
1887         case IP_VERSION(9, 1, 0):
1888                 adev->gfx.config.max_hw_contexts = 8;
1889                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1890                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1891                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1892                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1893                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1894                         gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
1895                 else
1896                         gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
1897                 break;
1898         case IP_VERSION(9, 4, 1):
1899                 adev->gfx.ras = &gfx_v9_4_ras;
1900                 adev->gfx.config.max_hw_contexts = 8;
1901                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1902                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1903                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1904                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1905                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1906                 gb_addr_config &= ~0xf3e777ff;
1907                 gb_addr_config |= 0x22014042;
1908                 break;
1909         case IP_VERSION(9, 3, 0):
1910                 adev->gfx.config.max_hw_contexts = 8;
1911                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1912                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1913                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
1914                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1915                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1916                 gb_addr_config &= ~0xf3e777ff;
1917                 gb_addr_config |= 0x22010042;
1918                 break;
1919         case IP_VERSION(9, 4, 2):
1920                 adev->gfx.ras = &gfx_v9_4_2_ras;
1921                 adev->gfx.config.max_hw_contexts = 8;
1922                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1923                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1924                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1925                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1926                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1927                 gb_addr_config &= ~0xf3e777ff;
1928                 gb_addr_config |= 0x22014042;
1929                 /* check vbios table if gpu info is not available */
1930                 err = amdgpu_atomfirmware_get_gfx_info(adev);
1931                 if (err)
1932                         return err;
1933                 break;
1934         default:
1935                 BUG();
1936                 break;
1937         }
1938
1939         adev->gfx.config.gb_addr_config = gb_addr_config;
1940
1941         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
1942                         REG_GET_FIELD(
1943                                         adev->gfx.config.gb_addr_config,
1944                                         GB_ADDR_CONFIG,
1945                                         NUM_PIPES);
1946
1947         adev->gfx.config.max_tile_pipes =
1948                 adev->gfx.config.gb_addr_config_fields.num_pipes;
1949
1950         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
1951                         REG_GET_FIELD(
1952                                         adev->gfx.config.gb_addr_config,
1953                                         GB_ADDR_CONFIG,
1954                                         NUM_BANKS);
1955         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
1956                         REG_GET_FIELD(
1957                                         adev->gfx.config.gb_addr_config,
1958                                         GB_ADDR_CONFIG,
1959                                         MAX_COMPRESSED_FRAGS);
1960         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
1961                         REG_GET_FIELD(
1962                                         adev->gfx.config.gb_addr_config,
1963                                         GB_ADDR_CONFIG,
1964                                         NUM_RB_PER_SE);
1965         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
1966                         REG_GET_FIELD(
1967                                         adev->gfx.config.gb_addr_config,
1968                                         GB_ADDR_CONFIG,
1969                                         NUM_SHADER_ENGINES);
1970         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
1971                         REG_GET_FIELD(
1972                                         adev->gfx.config.gb_addr_config,
1973                                         GB_ADDR_CONFIG,
1974                                         PIPE_INTERLEAVE_SIZE));
1975
1976         return 0;
1977 }
1978
1979 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1980                                       int mec, int pipe, int queue)
1981 {
1982         unsigned irq_type;
1983         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
1984         unsigned int hw_prio;
1985
1986         ring = &adev->gfx.compute_ring[ring_id];
1987
1988         /* mec0 is me1 */
1989         ring->me = mec + 1;
1990         ring->pipe = pipe;
1991         ring->queue = queue;
1992
1993         ring->ring_obj = NULL;
1994         ring->use_doorbell = true;
1995         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1996         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1997                                 + (ring_id * GFX9_MEC_HPD_SIZE);
1998         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1999
2000         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2001                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2002                 + ring->pipe;
2003         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2004                         AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
2005         /* type-2 packets are deprecated on MEC, use type-3 instead */
2006         return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2007                                 hw_prio, NULL);
2008 }
2009
2010 static int gfx_v9_0_sw_init(void *handle)
2011 {
2012         int i, j, k, r, ring_id;
2013         struct amdgpu_ring *ring;
2014         struct amdgpu_kiq *kiq;
2015         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2016         unsigned int hw_prio;
2017
2018         switch (adev->ip_versions[GC_HWIP][0]) {
2019         case IP_VERSION(9, 0, 1):
2020         case IP_VERSION(9, 2, 1):
2021         case IP_VERSION(9, 4, 0):
2022         case IP_VERSION(9, 2, 2):
2023         case IP_VERSION(9, 1, 0):
2024         case IP_VERSION(9, 4, 1):
2025         case IP_VERSION(9, 3, 0):
2026         case IP_VERSION(9, 4, 2):
2027                 adev->gfx.mec.num_mec = 2;
2028                 break;
2029         default:
2030                 adev->gfx.mec.num_mec = 1;
2031                 break;
2032         }
2033
2034         adev->gfx.mec.num_pipe_per_mec = 4;
2035         adev->gfx.mec.num_queue_per_pipe = 8;
2036
2037         /* EOP Event */
2038         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2039         if (r)
2040                 return r;
2041
2042         /* Privileged reg */
2043         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2044                               &adev->gfx.priv_reg_irq);
2045         if (r)
2046                 return r;
2047
2048         /* Privileged inst */
2049         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2050                               &adev->gfx.priv_inst_irq);
2051         if (r)
2052                 return r;
2053
2054         /* ECC error */
2055         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2056                               &adev->gfx.cp_ecc_error_irq);
2057         if (r)
2058                 return r;
2059
2060         /* FUE error */
2061         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2062                               &adev->gfx.cp_ecc_error_irq);
2063         if (r)
2064                 return r;
2065
2066         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2067
2068         if (adev->gfx.rlc.funcs) {
2069                 if (adev->gfx.rlc.funcs->init) {
2070                         r = adev->gfx.rlc.funcs->init(adev);
2071                         if (r) {
2072                                 dev_err(adev->dev, "Failed to init rlc BOs!\n");
2073                                 return r;
2074                         }
2075                 }
2076         }
2077
2078         r = gfx_v9_0_mec_init(adev);
2079         if (r) {
2080                 DRM_ERROR("Failed to init MEC BOs!\n");
2081                 return r;
2082         }
2083
2084         /* set up the gfx ring */
2085         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2086                 ring = &adev->gfx.gfx_ring[i];
2087                 ring->ring_obj = NULL;
2088                 if (!i)
2089                         sprintf(ring->name, "gfx");
2090                 else
2091                         sprintf(ring->name, "gfx_%d", i);
2092                 ring->use_doorbell = true;
2093                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2094
2095                 /* disable scheduler on the real ring */
2096                 ring->no_scheduler = true;
2097                 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2098                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2099                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
2100                 if (r)
2101                         return r;
2102         }
2103
2104         /* set up the software rings */
2105         if (adev->gfx.num_gfx_rings) {
2106                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2107                         ring = &adev->gfx.sw_gfx_ring[i];
2108                         ring->ring_obj = NULL;
2109                         sprintf(ring->name, amdgpu_sw_ring_name(i));
2110                         ring->use_doorbell = true;
2111                         ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2112                         ring->is_sw_ring = true;
2113                         hw_prio = amdgpu_sw_ring_priority(i);
2114                         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2115                                              AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,
2116                                              NULL);
2117                         if (r)
2118                                 return r;
2119                         ring->wptr = 0;
2120                 }
2121
2122                 /* init the muxer and add software rings */
2123                 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0],
2124                                          GFX9_NUM_SW_GFX_RINGS);
2125                 if (r) {
2126                         DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r);
2127                         return r;
2128                 }
2129                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2130                         r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer,
2131                                                         &adev->gfx.sw_gfx_ring[i]);
2132                         if (r) {
2133                                 DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r);
2134                                 return r;
2135                         }
2136                 }
2137         }
2138
2139         /* set up the compute queues - allocate horizontally across pipes */
2140         ring_id = 0;
2141         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2142                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2143                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2144                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2145                                         continue;
2146
2147                                 r = gfx_v9_0_compute_ring_init(adev,
2148                                                                ring_id,
2149                                                                i, k, j);
2150                                 if (r)
2151                                         return r;
2152
2153                                 ring_id++;
2154                         }
2155                 }
2156         }
2157
2158         r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2159         if (r) {
2160                 DRM_ERROR("Failed to init KIQ BOs!\n");
2161                 return r;
2162         }
2163
2164         kiq = &adev->gfx.kiq;
2165         r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2166         if (r)
2167                 return r;
2168
2169         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2170         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2171         if (r)
2172                 return r;
2173
2174         adev->gfx.ce_ram_size = 0x8000;
2175
2176         r = gfx_v9_0_gpu_early_init(adev);
2177         if (r)
2178                 return r;
2179
2180         if (amdgpu_gfx_ras_sw_init(adev)) {
2181                 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
2182                 return -EINVAL;
2183         }
2184
2185         return 0;
2186 }
2187
2188
2189 static int gfx_v9_0_sw_fini(void *handle)
2190 {
2191         int i;
2192         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2193
2194         if (adev->gfx.num_gfx_rings) {
2195                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
2196                         amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]);
2197                 amdgpu_ring_mux_fini(&adev->gfx.muxer);
2198         }
2199
2200         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2201                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2202         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2203                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2204
2205         amdgpu_gfx_mqd_sw_fini(adev);
2206         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2207         amdgpu_gfx_kiq_fini(adev);
2208
2209         gfx_v9_0_mec_fini(adev);
2210         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2211                                 &adev->gfx.rlc.clear_state_gpu_addr,
2212                                 (void **)&adev->gfx.rlc.cs_ptr);
2213         if (adev->flags & AMD_IS_APU) {
2214                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2215                                 &adev->gfx.rlc.cp_table_gpu_addr,
2216                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2217         }
2218         gfx_v9_0_free_microcode(adev);
2219
2220         return 0;
2221 }
2222
2223
2224 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2225 {
2226         /* TODO */
2227 }
2228
2229 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2230                            u32 instance)
2231 {
2232         u32 data;
2233
2234         if (instance == 0xffffffff)
2235                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2236         else
2237                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2238
2239         if (se_num == 0xffffffff)
2240                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2241         else
2242                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2243
2244         if (sh_num == 0xffffffff)
2245                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2246         else
2247                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2248
2249         WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2250 }
2251
2252 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2253 {
2254         u32 data, mask;
2255
2256         data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2257         data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2258
2259         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2260         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2261
2262         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2263                                          adev->gfx.config.max_sh_per_se);
2264
2265         return (~data) & mask;
2266 }
2267
2268 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2269 {
2270         int i, j;
2271         u32 data;
2272         u32 active_rbs = 0;
2273         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2274                                         adev->gfx.config.max_sh_per_se;
2275
2276         mutex_lock(&adev->grbm_idx_mutex);
2277         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2278                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2279                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff);
2280                         data = gfx_v9_0_get_rb_active_bitmap(adev);
2281                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2282                                                rb_bitmap_width_per_sh);
2283                 }
2284         }
2285         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2286         mutex_unlock(&adev->grbm_idx_mutex);
2287
2288         adev->gfx.config.backend_enable_mask = active_rbs;
2289         adev->gfx.config.num_rbs = hweight32(active_rbs);
2290 }
2291
2292 #define DEFAULT_SH_MEM_BASES    (0x6000)
2293 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2294 {
2295         int i;
2296         uint32_t sh_mem_config;
2297         uint32_t sh_mem_bases;
2298
2299         /*
2300          * Configure apertures:
2301          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
2302          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
2303          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
2304          */
2305         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2306
2307         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2308                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2309                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2310
2311         mutex_lock(&adev->srbm_mutex);
2312         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2313                 soc15_grbm_select(adev, 0, 0, 0, i);
2314                 /* CP and shaders */
2315                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2316                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2317         }
2318         soc15_grbm_select(adev, 0, 0, 0, 0);
2319         mutex_unlock(&adev->srbm_mutex);
2320
2321         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2322            access. These should be enabled by FW for target VMIDs. */
2323         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2324                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2325                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2326                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2327                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2328         }
2329 }
2330
2331 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2332 {
2333         int vmid;
2334
2335         /*
2336          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2337          * access. Compute VMIDs should be enabled by FW for target VMIDs,
2338          * the driver can enable them for graphics. VMID0 should maintain
2339          * access so that HWS firmware can save/restore entries.
2340          */
2341         for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2342                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2343                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2344                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2345                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2346         }
2347 }
2348
2349 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2350 {
2351         uint32_t tmp;
2352
2353         switch (adev->ip_versions[GC_HWIP][0]) {
2354         case IP_VERSION(9, 4, 1):
2355                 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2356                 tmp = REG_SET_FIELD(tmp, SQ_CONFIG,
2357                                         DISABLE_BARRIER_WAITCNT, 1);
2358                 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2359                 break;
2360         default:
2361                 break;
2362         }
2363 }
2364
2365 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2366 {
2367         u32 tmp;
2368         int i;
2369
2370         WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2371
2372         gfx_v9_0_tiling_mode_table_init(adev);
2373
2374         if (adev->gfx.num_gfx_rings)
2375                 gfx_v9_0_setup_rb(adev);
2376         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2377         adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2378
2379         /* XXX SH_MEM regs */
2380         /* where to put LDS, scratch, GPUVM in FSA64 space */
2381         mutex_lock(&adev->srbm_mutex);
2382         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2383                 soc15_grbm_select(adev, 0, 0, 0, i);
2384                 /* CP and shaders */
2385                 if (i == 0) {
2386                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2387                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2388                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2389                                             !!adev->gmc.noretry);
2390                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2391                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2392                 } else {
2393                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2394                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2395                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2396                                             !!adev->gmc.noretry);
2397                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2398                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2399                                 (adev->gmc.private_aperture_start >> 48));
2400                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2401                                 (adev->gmc.shared_aperture_start >> 48));
2402                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2403                 }
2404         }
2405         soc15_grbm_select(adev, 0, 0, 0, 0);
2406
2407         mutex_unlock(&adev->srbm_mutex);
2408
2409         gfx_v9_0_init_compute_vmid(adev);
2410         gfx_v9_0_init_gds_vmid(adev);
2411         gfx_v9_0_init_sq_config(adev);
2412 }
2413
2414 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2415 {
2416         u32 i, j, k;
2417         u32 mask;
2418
2419         mutex_lock(&adev->grbm_idx_mutex);
2420         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2421                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2422                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff);
2423                         for (k = 0; k < adev->usec_timeout; k++) {
2424                                 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2425                                         break;
2426                                 udelay(1);
2427                         }
2428                         if (k == adev->usec_timeout) {
2429                                 amdgpu_gfx_select_se_sh(adev, 0xffffffff,
2430                                                       0xffffffff, 0xffffffff);
2431                                 mutex_unlock(&adev->grbm_idx_mutex);
2432                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2433                                          i, j);
2434                                 return;
2435                         }
2436                 }
2437         }
2438         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2439         mutex_unlock(&adev->grbm_idx_mutex);
2440
2441         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2442                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2443                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2444                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2445         for (k = 0; k < adev->usec_timeout; k++) {
2446                 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2447                         break;
2448                 udelay(1);
2449         }
2450 }
2451
2452 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2453                                                bool enable)
2454 {
2455         u32 tmp;
2456
2457         /* These interrupts should be enabled to drive DS clock */
2458
2459         tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2460
2461         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2462         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2463         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2464         if(adev->gfx.num_gfx_rings)
2465                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2466
2467         WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2468 }
2469
2470 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2471 {
2472         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2473         /* csib */
2474         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2475                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
2476         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2477                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2478         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2479                         adev->gfx.rlc.clear_state_size);
2480 }
2481
2482 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2483                                 int indirect_offset,
2484                                 int list_size,
2485                                 int *unique_indirect_regs,
2486                                 int unique_indirect_reg_count,
2487                                 int *indirect_start_offsets,
2488                                 int *indirect_start_offsets_count,
2489                                 int max_start_offsets_count)
2490 {
2491         int idx;
2492
2493         for (; indirect_offset < list_size; indirect_offset++) {
2494                 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2495                 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2496                 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2497
2498                 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2499                         indirect_offset += 2;
2500
2501                         /* look for the matching indice */
2502                         for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2503                                 if (unique_indirect_regs[idx] ==
2504                                         register_list_format[indirect_offset] ||
2505                                         !unique_indirect_regs[idx])
2506                                         break;
2507                         }
2508
2509                         BUG_ON(idx >= unique_indirect_reg_count);
2510
2511                         if (!unique_indirect_regs[idx])
2512                                 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2513
2514                         indirect_offset++;
2515                 }
2516         }
2517 }
2518
2519 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2520 {
2521         int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2522         int unique_indirect_reg_count = 0;
2523
2524         int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2525         int indirect_start_offsets_count = 0;
2526
2527         int list_size = 0;
2528         int i = 0, j = 0;
2529         u32 tmp = 0;
2530
2531         u32 *register_list_format =
2532                 kmemdup(adev->gfx.rlc.register_list_format,
2533                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2534         if (!register_list_format)
2535                 return -ENOMEM;
2536
2537         /* setup unique_indirect_regs array and indirect_start_offsets array */
2538         unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2539         gfx_v9_1_parse_ind_reg_list(register_list_format,
2540                                     adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2541                                     adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2542                                     unique_indirect_regs,
2543                                     unique_indirect_reg_count,
2544                                     indirect_start_offsets,
2545                                     &indirect_start_offsets_count,
2546                                     ARRAY_SIZE(indirect_start_offsets));
2547
2548         /* enable auto inc in case it is disabled */
2549         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2550         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2551         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2552
2553         /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2554         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2555                 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2556         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2557                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2558                         adev->gfx.rlc.register_restore[i]);
2559
2560         /* load indirect register */
2561         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2562                 adev->gfx.rlc.reg_list_format_start);
2563
2564         /* direct register portion */
2565         for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2566                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2567                         register_list_format[i]);
2568
2569         /* indirect register portion */
2570         while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2571                 if (register_list_format[i] == 0xFFFFFFFF) {
2572                         WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2573                         continue;
2574                 }
2575
2576                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2577                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2578
2579                 for (j = 0; j < unique_indirect_reg_count; j++) {
2580                         if (register_list_format[i] == unique_indirect_regs[j]) {
2581                                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2582                                 break;
2583                         }
2584                 }
2585
2586                 BUG_ON(j >= unique_indirect_reg_count);
2587
2588                 i++;
2589         }
2590
2591         /* set save/restore list size */
2592         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2593         list_size = list_size >> 1;
2594         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2595                 adev->gfx.rlc.reg_restore_list_size);
2596         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2597
2598         /* write the starting offsets to RLC scratch ram */
2599         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2600                 adev->gfx.rlc.starting_offsets_start);
2601         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2602                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2603                        indirect_start_offsets[i]);
2604
2605         /* load unique indirect regs*/
2606         for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2607                 if (unique_indirect_regs[i] != 0) {
2608                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2609                                + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2610                                unique_indirect_regs[i] & 0x3FFFF);
2611
2612                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2613                                + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2614                                unique_indirect_regs[i] >> 20);
2615                 }
2616         }
2617
2618         kfree(register_list_format);
2619         return 0;
2620 }
2621
2622 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2623 {
2624         WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2625 }
2626
2627 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2628                                              bool enable)
2629 {
2630         uint32_t data = 0;
2631         uint32_t default_data = 0;
2632
2633         default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2634         if (enable) {
2635                 /* enable GFXIP control over CGPG */
2636                 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2637                 if(default_data != data)
2638                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2639
2640                 /* update status */
2641                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2642                 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2643                 if(default_data != data)
2644                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2645         } else {
2646                 /* restore GFXIP control over GCPG */
2647                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2648                 if(default_data != data)
2649                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2650         }
2651 }
2652
2653 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2654 {
2655         uint32_t data = 0;
2656
2657         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2658                               AMD_PG_SUPPORT_GFX_SMG |
2659                               AMD_PG_SUPPORT_GFX_DMG)) {
2660                 /* init IDLE_POLL_COUNT = 60 */
2661                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2662                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2663                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2664                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2665
2666                 /* init RLC PG Delay */
2667                 data = 0;
2668                 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2669                 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2670                 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2671                 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2672                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2673
2674                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2675                 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2676                 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2677                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2678
2679                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2680                 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2681                 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2682                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2683
2684                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2685                 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2686
2687                 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2688                 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2689                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2690                 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 3, 0))
2691                         pwr_10_0_gfxip_control_over_cgpg(adev, true);
2692         }
2693 }
2694
2695 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2696                                                 bool enable)
2697 {
2698         uint32_t data = 0;
2699         uint32_t default_data = 0;
2700
2701         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2702         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2703                              SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2704                              enable ? 1 : 0);
2705         if (default_data != data)
2706                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2707 }
2708
2709 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2710                                                 bool enable)
2711 {
2712         uint32_t data = 0;
2713         uint32_t default_data = 0;
2714
2715         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2716         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2717                              SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2718                              enable ? 1 : 0);
2719         if(default_data != data)
2720                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2721 }
2722
2723 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2724                                         bool enable)
2725 {
2726         uint32_t data = 0;
2727         uint32_t default_data = 0;
2728
2729         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2730         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2731                              CP_PG_DISABLE,
2732                              enable ? 0 : 1);
2733         if(default_data != data)
2734                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2735 }
2736
2737 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2738                                                 bool enable)
2739 {
2740         uint32_t data, default_data;
2741
2742         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2743         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2744                              GFX_POWER_GATING_ENABLE,
2745                              enable ? 1 : 0);
2746         if(default_data != data)
2747                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2748 }
2749
2750 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2751                                                 bool enable)
2752 {
2753         uint32_t data, default_data;
2754
2755         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2756         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2757                              GFX_PIPELINE_PG_ENABLE,
2758                              enable ? 1 : 0);
2759         if(default_data != data)
2760                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2761
2762         if (!enable)
2763                 /* read any GFX register to wake up GFX */
2764                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2765 }
2766
2767 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
2768                                                        bool enable)
2769 {
2770         uint32_t data, default_data;
2771
2772         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2773         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2774                              STATIC_PER_CU_PG_ENABLE,
2775                              enable ? 1 : 0);
2776         if(default_data != data)
2777                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2778 }
2779
2780 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
2781                                                 bool enable)
2782 {
2783         uint32_t data, default_data;
2784
2785         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2786         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2787                              DYN_PER_CU_PG_ENABLE,
2788                              enable ? 1 : 0);
2789         if(default_data != data)
2790                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2791 }
2792
2793 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
2794 {
2795         gfx_v9_0_init_csb(adev);
2796
2797         /*
2798          * Rlc save restore list is workable since v2_1.
2799          * And it's needed by gfxoff feature.
2800          */
2801         if (adev->gfx.rlc.is_rlc_v2_1) {
2802                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 2, 1) ||
2803                     (adev->apu_flags & AMD_APU_IS_RAVEN2))
2804                         gfx_v9_1_init_rlc_save_restore_list(adev);
2805                 gfx_v9_0_enable_save_restore_machine(adev);
2806         }
2807
2808         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2809                               AMD_PG_SUPPORT_GFX_SMG |
2810                               AMD_PG_SUPPORT_GFX_DMG |
2811                               AMD_PG_SUPPORT_CP |
2812                               AMD_PG_SUPPORT_GDS |
2813                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
2814                 WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,
2815                              adev->gfx.rlc.cp_table_gpu_addr >> 8);
2816                 gfx_v9_0_init_gfx_power_gating(adev);
2817         }
2818 }
2819
2820 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
2821 {
2822         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
2823         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
2824         gfx_v9_0_wait_for_rlc_serdes(adev);
2825 }
2826
2827 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
2828 {
2829         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2830         udelay(50);
2831         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2832         udelay(50);
2833 }
2834
2835 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
2836 {
2837 #ifdef AMDGPU_RLC_DEBUG_RETRY
2838         u32 rlc_ucode_ver;
2839 #endif
2840
2841         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2842         udelay(50);
2843
2844         /* carrizo do enable cp interrupt after cp inited */
2845         if (!(adev->flags & AMD_IS_APU)) {
2846                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
2847                 udelay(50);
2848         }
2849
2850 #ifdef AMDGPU_RLC_DEBUG_RETRY
2851         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
2852         rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
2853         if(rlc_ucode_ver == 0x108) {
2854                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
2855                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
2856                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
2857                  * default is 0x9C4 to create a 100us interval */
2858                 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
2859                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
2860                  * to disable the page fault retry interrupts, default is
2861                  * 0x100 (256) */
2862                 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
2863         }
2864 #endif
2865 }
2866
2867 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
2868 {
2869         const struct rlc_firmware_header_v2_0 *hdr;
2870         const __le32 *fw_data;
2871         unsigned i, fw_size;
2872
2873         if (!adev->gfx.rlc_fw)
2874                 return -EINVAL;
2875
2876         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2877         amdgpu_ucode_print_rlc_hdr(&hdr->header);
2878
2879         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2880                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2881         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2882
2883         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
2884                         RLCG_UCODE_LOADING_START_ADDRESS);
2885         for (i = 0; i < fw_size; i++)
2886                 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
2887         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2888
2889         return 0;
2890 }
2891
2892 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
2893 {
2894         int r;
2895
2896         if (amdgpu_sriov_vf(adev)) {
2897                 gfx_v9_0_init_csb(adev);
2898                 return 0;
2899         }
2900
2901         adev->gfx.rlc.funcs->stop(adev);
2902
2903         /* disable CG */
2904         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
2905
2906         gfx_v9_0_init_pg(adev);
2907
2908         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
2909                 /* legacy rlc firmware loading */
2910                 r = gfx_v9_0_rlc_load_microcode(adev);
2911                 if (r)
2912                         return r;
2913         }
2914
2915         switch (adev->ip_versions[GC_HWIP][0]) {
2916         case IP_VERSION(9, 2, 2):
2917         case IP_VERSION(9, 1, 0):
2918                 if (amdgpu_lbpw == 0)
2919                         gfx_v9_0_enable_lbpw(adev, false);
2920                 else
2921                         gfx_v9_0_enable_lbpw(adev, true);
2922                 break;
2923         case IP_VERSION(9, 4, 0):
2924                 if (amdgpu_lbpw > 0)
2925                         gfx_v9_0_enable_lbpw(adev, true);
2926                 else
2927                         gfx_v9_0_enable_lbpw(adev, false);
2928                 break;
2929         default:
2930                 break;
2931         }
2932
2933         adev->gfx.rlc.funcs->start(adev);
2934
2935         return 0;
2936 }
2937
2938 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2939 {
2940         u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
2941
2942         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2943         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2944         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
2945         WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
2946         udelay(50);
2947 }
2948
2949 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2950 {
2951         const struct gfx_firmware_header_v1_0 *pfp_hdr;
2952         const struct gfx_firmware_header_v1_0 *ce_hdr;
2953         const struct gfx_firmware_header_v1_0 *me_hdr;
2954         const __le32 *fw_data;
2955         unsigned i, fw_size;
2956
2957         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
2958                 return -EINVAL;
2959
2960         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2961                 adev->gfx.pfp_fw->data;
2962         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
2963                 adev->gfx.ce_fw->data;
2964         me_hdr = (const struct gfx_firmware_header_v1_0 *)
2965                 adev->gfx.me_fw->data;
2966
2967         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2968         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
2969         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2970
2971         gfx_v9_0_cp_gfx_enable(adev, false);
2972
2973         /* PFP */
2974         fw_data = (const __le32 *)
2975                 (adev->gfx.pfp_fw->data +
2976                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2977         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
2978         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
2979         for (i = 0; i < fw_size; i++)
2980                 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
2981         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2982
2983         /* CE */
2984         fw_data = (const __le32 *)
2985                 (adev->gfx.ce_fw->data +
2986                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
2987         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
2988         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
2989         for (i = 0; i < fw_size; i++)
2990                 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
2991         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
2992
2993         /* ME */
2994         fw_data = (const __le32 *)
2995                 (adev->gfx.me_fw->data +
2996                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
2997         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
2998         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
2999         for (i = 0; i < fw_size; i++)
3000                 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3001         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3002
3003         return 0;
3004 }
3005
3006 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3007 {
3008         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3009         const struct cs_section_def *sect = NULL;
3010         const struct cs_extent_def *ext = NULL;
3011         int r, i, tmp;
3012
3013         /* init the CP */
3014         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3015         WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3016
3017         gfx_v9_0_cp_gfx_enable(adev, true);
3018
3019         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3020         if (r) {
3021                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3022                 return r;
3023         }
3024
3025         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3026         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3027
3028         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3029         amdgpu_ring_write(ring, 0x80000000);
3030         amdgpu_ring_write(ring, 0x80000000);
3031
3032         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3033                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3034                         if (sect->id == SECT_CONTEXT) {
3035                                 amdgpu_ring_write(ring,
3036                                        PACKET3(PACKET3_SET_CONTEXT_REG,
3037                                                ext->reg_count));
3038                                 amdgpu_ring_write(ring,
3039                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3040                                 for (i = 0; i < ext->reg_count; i++)
3041                                         amdgpu_ring_write(ring, ext->extent[i]);
3042                         }
3043                 }
3044         }
3045
3046         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3047         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3048
3049         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3050         amdgpu_ring_write(ring, 0);
3051
3052         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3053         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3054         amdgpu_ring_write(ring, 0x8000);
3055         amdgpu_ring_write(ring, 0x8000);
3056
3057         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3058         tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3059                 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3060         amdgpu_ring_write(ring, tmp);
3061         amdgpu_ring_write(ring, 0);
3062
3063         amdgpu_ring_commit(ring);
3064
3065         return 0;
3066 }
3067
3068 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3069 {
3070         struct amdgpu_ring *ring;
3071         u32 tmp;
3072         u32 rb_bufsz;
3073         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3074
3075         /* Set the write pointer delay */
3076         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3077
3078         /* set the RB to use vmid 0 */
3079         WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3080
3081         /* Set ring buffer size */
3082         ring = &adev->gfx.gfx_ring[0];
3083         rb_bufsz = order_base_2(ring->ring_size / 8);
3084         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3085         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3086 #ifdef __BIG_ENDIAN
3087         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3088 #endif
3089         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3090
3091         /* Initialize the ring buffer's write pointers */
3092         ring->wptr = 0;
3093         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3094         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3095
3096         /* set the wb address wether it's enabled or not */
3097         rptr_addr = ring->rptr_gpu_addr;
3098         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3099         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3100
3101         wptr_gpu_addr = ring->wptr_gpu_addr;
3102         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3103         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3104
3105         mdelay(1);
3106         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3107
3108         rb_addr = ring->gpu_addr >> 8;
3109         WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3110         WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3111
3112         tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3113         if (ring->use_doorbell) {
3114                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3115                                     DOORBELL_OFFSET, ring->doorbell_index);
3116                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3117                                     DOORBELL_EN, 1);
3118         } else {
3119                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3120         }
3121         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3122
3123         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3124                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
3125         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3126
3127         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3128                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3129
3130
3131         /* start the ring */
3132         gfx_v9_0_cp_gfx_start(adev);
3133         ring->sched.ready = true;
3134
3135         return 0;
3136 }
3137
3138 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3139 {
3140         if (enable) {
3141                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3142         } else {
3143                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3144                         (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3145                 adev->gfx.kiq.ring.sched.ready = false;
3146         }
3147         udelay(50);
3148 }
3149
3150 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3151 {
3152         const struct gfx_firmware_header_v1_0 *mec_hdr;
3153         const __le32 *fw_data;
3154         unsigned i;
3155         u32 tmp;
3156
3157         if (!adev->gfx.mec_fw)
3158                 return -EINVAL;
3159
3160         gfx_v9_0_cp_compute_enable(adev, false);
3161
3162         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3163         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3164
3165         fw_data = (const __le32 *)
3166                 (adev->gfx.mec_fw->data +
3167                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3168         tmp = 0;
3169         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3170         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3171         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3172
3173         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3174                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3175         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3176                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3177
3178         /* MEC1 */
3179         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3180                          mec_hdr->jt_offset);
3181         for (i = 0; i < mec_hdr->jt_size; i++)
3182                 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3183                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3184
3185         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3186                         adev->gfx.mec_fw_version);
3187         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3188
3189         return 0;
3190 }
3191
3192 /* KIQ functions */
3193 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3194 {
3195         uint32_t tmp;
3196         struct amdgpu_device *adev = ring->adev;
3197
3198         /* tell RLC which is KIQ queue */
3199         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3200         tmp &= 0xffffff00;
3201         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3202         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3203         tmp |= 0x80;
3204         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3205 }
3206
3207 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3208 {
3209         struct amdgpu_device *adev = ring->adev;
3210
3211         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3212                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3213                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3214                         mqd->cp_hqd_queue_priority =
3215                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3216                 }
3217         }
3218 }
3219
3220 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3221 {
3222         struct amdgpu_device *adev = ring->adev;
3223         struct v9_mqd *mqd = ring->mqd_ptr;
3224         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3225         uint32_t tmp;
3226
3227         mqd->header = 0xC0310800;
3228         mqd->compute_pipelinestat_enable = 0x00000001;
3229         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3230         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3231         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3232         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3233         mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3234         mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3235         mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3236         mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3237         mqd->compute_misc_reserved = 0x00000003;
3238
3239         mqd->dynamic_cu_mask_addr_lo =
3240                 lower_32_bits(ring->mqd_gpu_addr
3241                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3242         mqd->dynamic_cu_mask_addr_hi =
3243                 upper_32_bits(ring->mqd_gpu_addr
3244                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3245
3246         eop_base_addr = ring->eop_gpu_addr >> 8;
3247         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3248         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3249
3250         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3251         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3252         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3253                         (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3254
3255         mqd->cp_hqd_eop_control = tmp;
3256
3257         /* enable doorbell? */
3258         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3259
3260         if (ring->use_doorbell) {
3261                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3262                                     DOORBELL_OFFSET, ring->doorbell_index);
3263                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3264                                     DOORBELL_EN, 1);
3265                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3266                                     DOORBELL_SOURCE, 0);
3267                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3268                                     DOORBELL_HIT, 0);
3269         } else {
3270                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3271                                          DOORBELL_EN, 0);
3272         }
3273
3274         mqd->cp_hqd_pq_doorbell_control = tmp;
3275
3276         /* disable the queue if it's active */
3277         ring->wptr = 0;
3278         mqd->cp_hqd_dequeue_request = 0;
3279         mqd->cp_hqd_pq_rptr = 0;
3280         mqd->cp_hqd_pq_wptr_lo = 0;
3281         mqd->cp_hqd_pq_wptr_hi = 0;
3282
3283         /* set the pointer to the MQD */
3284         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3285         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3286
3287         /* set MQD vmid to 0 */
3288         tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3289         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3290         mqd->cp_mqd_control = tmp;
3291
3292         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3293         hqd_gpu_addr = ring->gpu_addr >> 8;
3294         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3295         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3296
3297         /* set up the HQD, this is similar to CP_RB0_CNTL */
3298         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3299         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3300                             (order_base_2(ring->ring_size / 4) - 1));
3301         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3302                         (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3303 #ifdef __BIG_ENDIAN
3304         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3305 #endif
3306         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3307         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3308         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3309         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3310         mqd->cp_hqd_pq_control = tmp;
3311
3312         /* set the wb address whether it's enabled or not */
3313         wb_gpu_addr = ring->rptr_gpu_addr;
3314         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3315         mqd->cp_hqd_pq_rptr_report_addr_hi =
3316                 upper_32_bits(wb_gpu_addr) & 0xffff;
3317
3318         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3319         wb_gpu_addr = ring->wptr_gpu_addr;
3320         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3321         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3322
3323         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3324         ring->wptr = 0;
3325         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3326
3327         /* set the vmid for the queue */
3328         mqd->cp_hqd_vmid = 0;
3329
3330         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3331         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3332         mqd->cp_hqd_persistent_state = tmp;
3333
3334         /* set MIN_IB_AVAIL_SIZE */
3335         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3336         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3337         mqd->cp_hqd_ib_control = tmp;
3338
3339         /* set static priority for a queue/ring */
3340         gfx_v9_0_mqd_set_priority(ring, mqd);
3341         mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
3342
3343         /* map_queues packet doesn't need activate the queue,
3344          * so only kiq need set this field.
3345          */
3346         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3347                 mqd->cp_hqd_active = 1;
3348
3349         return 0;
3350 }
3351
3352 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3353 {
3354         struct amdgpu_device *adev = ring->adev;
3355         struct v9_mqd *mqd = ring->mqd_ptr;
3356         int j;
3357
3358         /* disable wptr polling */
3359         WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3360
3361         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3362                mqd->cp_hqd_eop_base_addr_lo);
3363         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3364                mqd->cp_hqd_eop_base_addr_hi);
3365
3366         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3367         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3368                mqd->cp_hqd_eop_control);
3369
3370         /* enable doorbell? */
3371         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3372                mqd->cp_hqd_pq_doorbell_control);
3373
3374         /* disable the queue if it's active */
3375         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3376                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3377                 for (j = 0; j < adev->usec_timeout; j++) {
3378                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3379                                 break;
3380                         udelay(1);
3381                 }
3382                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3383                        mqd->cp_hqd_dequeue_request);
3384                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3385                        mqd->cp_hqd_pq_rptr);
3386                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3387                        mqd->cp_hqd_pq_wptr_lo);
3388                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3389                        mqd->cp_hqd_pq_wptr_hi);
3390         }
3391
3392         /* set the pointer to the MQD */
3393         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3394                mqd->cp_mqd_base_addr_lo);
3395         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3396                mqd->cp_mqd_base_addr_hi);
3397
3398         /* set MQD vmid to 0 */
3399         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3400                mqd->cp_mqd_control);
3401
3402         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3403         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3404                mqd->cp_hqd_pq_base_lo);
3405         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3406                mqd->cp_hqd_pq_base_hi);
3407
3408         /* set up the HQD, this is similar to CP_RB0_CNTL */
3409         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3410                mqd->cp_hqd_pq_control);
3411
3412         /* set the wb address whether it's enabled or not */
3413         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3414                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3415         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3416                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3417
3418         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3419         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3420                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3421         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3422                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3423
3424         /* enable the doorbell if requested */
3425         if (ring->use_doorbell) {
3426                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3427                                         (adev->doorbell_index.kiq * 2) << 2);
3428                 /* If GC has entered CGPG, ringing doorbell > first page
3429                  * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3430                  * workaround this issue. And this change has to align with firmware
3431                  * update.
3432                  */
3433                 if (check_if_enlarge_doorbell_range(adev))
3434                         WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3435                                         (adev->doorbell.size - 4));
3436                 else
3437                         WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3438                                         (adev->doorbell_index.userqueue_end * 2) << 2);
3439         }
3440
3441         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3442                mqd->cp_hqd_pq_doorbell_control);
3443
3444         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3445         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3446                mqd->cp_hqd_pq_wptr_lo);
3447         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3448                mqd->cp_hqd_pq_wptr_hi);
3449
3450         /* set the vmid for the queue */
3451         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3452
3453         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3454                mqd->cp_hqd_persistent_state);
3455
3456         /* activate the queue */
3457         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3458                mqd->cp_hqd_active);
3459
3460         if (ring->use_doorbell)
3461                 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3462
3463         return 0;
3464 }
3465
3466 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3467 {
3468         struct amdgpu_device *adev = ring->adev;
3469         int j;
3470
3471         /* disable the queue if it's active */
3472         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3473
3474                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3475
3476                 for (j = 0; j < adev->usec_timeout; j++) {
3477                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3478                                 break;
3479                         udelay(1);
3480                 }
3481
3482                 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3483                         DRM_DEBUG("KIQ dequeue request failed.\n");
3484
3485                         /* Manual disable if dequeue request times out */
3486                         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3487                 }
3488
3489                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3490                       0);
3491         }
3492
3493         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3494         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3495         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3496         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3497         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3498         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3499         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3500         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3501
3502         return 0;
3503 }
3504
3505 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3506 {
3507         struct amdgpu_device *adev = ring->adev;
3508         struct v9_mqd *mqd = ring->mqd_ptr;
3509         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3510         struct v9_mqd *tmp_mqd;
3511
3512         gfx_v9_0_kiq_setting(ring);
3513
3514         /* GPU could be in bad state during probe, driver trigger the reset
3515          * after load the SMU, in this case , the mqd is not be initialized.
3516          * driver need to re-init the mqd.
3517          * check mqd->cp_hqd_pq_control since this value should not be 0
3518          */
3519         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3520         if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3521                 /* for GPU_RESET case , reset MQD to a clean status */
3522                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3523                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3524
3525                 /* reset ring buffer */
3526                 ring->wptr = 0;
3527                 amdgpu_ring_clear_ring(ring);
3528
3529                 mutex_lock(&adev->srbm_mutex);
3530                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3531                 gfx_v9_0_kiq_init_register(ring);
3532                 soc15_grbm_select(adev, 0, 0, 0, 0);
3533                 mutex_unlock(&adev->srbm_mutex);
3534         } else {
3535                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3536                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3537                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3538                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3539                         amdgpu_ring_clear_ring(ring);
3540                 mutex_lock(&adev->srbm_mutex);
3541                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3542                 gfx_v9_0_mqd_init(ring);
3543                 gfx_v9_0_kiq_init_register(ring);
3544                 soc15_grbm_select(adev, 0, 0, 0, 0);
3545                 mutex_unlock(&adev->srbm_mutex);
3546
3547                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3548                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3549         }
3550
3551         return 0;
3552 }
3553
3554 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3555 {
3556         struct amdgpu_device *adev = ring->adev;
3557         struct v9_mqd *mqd = ring->mqd_ptr;
3558         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3559         struct v9_mqd *tmp_mqd;
3560
3561         /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3562          * is not be initialized before
3563          */
3564         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3565
3566         if (!tmp_mqd->cp_hqd_pq_control ||
3567             (!amdgpu_in_reset(adev) && !adev->in_suspend)) {
3568                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3569                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3570                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3571                 mutex_lock(&adev->srbm_mutex);
3572                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3573                 gfx_v9_0_mqd_init(ring);
3574                 soc15_grbm_select(adev, 0, 0, 0, 0);
3575                 mutex_unlock(&adev->srbm_mutex);
3576
3577                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3578                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3579         } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
3580                 /* reset MQD to a clean status */
3581                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3582                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3583
3584                 /* reset ring buffer */
3585                 ring->wptr = 0;
3586                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
3587                 amdgpu_ring_clear_ring(ring);
3588         } else {
3589                 amdgpu_ring_clear_ring(ring);
3590         }
3591
3592         return 0;
3593 }
3594
3595 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3596 {
3597         struct amdgpu_ring *ring;
3598         int r;
3599
3600         ring = &adev->gfx.kiq.ring;
3601
3602         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3603         if (unlikely(r != 0))
3604                 return r;
3605
3606         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3607         if (unlikely(r != 0))
3608                 return r;
3609
3610         gfx_v9_0_kiq_init_queue(ring);
3611         amdgpu_bo_kunmap(ring->mqd_obj);
3612         ring->mqd_ptr = NULL;
3613         amdgpu_bo_unreserve(ring->mqd_obj);
3614         ring->sched.ready = true;
3615         return 0;
3616 }
3617
3618 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3619 {
3620         struct amdgpu_ring *ring = NULL;
3621         int r = 0, i;
3622
3623         gfx_v9_0_cp_compute_enable(adev, true);
3624
3625         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3626                 ring = &adev->gfx.compute_ring[i];
3627
3628                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3629                 if (unlikely(r != 0))
3630                         goto done;
3631                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3632                 if (!r) {
3633                         r = gfx_v9_0_kcq_init_queue(ring);
3634                         amdgpu_bo_kunmap(ring->mqd_obj);
3635                         ring->mqd_ptr = NULL;
3636                 }
3637                 amdgpu_bo_unreserve(ring->mqd_obj);
3638                 if (r)
3639                         goto done;
3640         }
3641
3642         r = amdgpu_gfx_enable_kcq(adev);
3643 done:
3644         return r;
3645 }
3646
3647 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3648 {
3649         int r, i;
3650         struct amdgpu_ring *ring;
3651
3652         if (!(adev->flags & AMD_IS_APU))
3653                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3654
3655         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3656                 if (adev->gfx.num_gfx_rings) {
3657                         /* legacy firmware loading */
3658                         r = gfx_v9_0_cp_gfx_load_microcode(adev);
3659                         if (r)
3660                                 return r;
3661                 }
3662
3663                 r = gfx_v9_0_cp_compute_load_microcode(adev);
3664                 if (r)
3665                         return r;
3666         }
3667
3668         r = gfx_v9_0_kiq_resume(adev);
3669         if (r)
3670                 return r;
3671
3672         if (adev->gfx.num_gfx_rings) {
3673                 r = gfx_v9_0_cp_gfx_resume(adev);
3674                 if (r)
3675                         return r;
3676         }
3677
3678         r = gfx_v9_0_kcq_resume(adev);
3679         if (r)
3680                 return r;
3681
3682         if (adev->gfx.num_gfx_rings) {
3683                 ring = &adev->gfx.gfx_ring[0];
3684                 r = amdgpu_ring_test_helper(ring);
3685                 if (r)
3686                         return r;
3687         }
3688
3689         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3690                 ring = &adev->gfx.compute_ring[i];
3691                 amdgpu_ring_test_helper(ring);
3692         }
3693
3694         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3695
3696         return 0;
3697 }
3698
3699 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3700 {
3701         u32 tmp;
3702
3703         if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1) &&
3704             adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 2))
3705                 return;
3706
3707         tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3708         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3709                                 adev->df.hash_status.hash_64k);
3710         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3711                                 adev->df.hash_status.hash_2m);
3712         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3713                                 adev->df.hash_status.hash_1g);
3714         WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3715 }
3716
3717 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3718 {
3719         if (adev->gfx.num_gfx_rings)
3720                 gfx_v9_0_cp_gfx_enable(adev, enable);
3721         gfx_v9_0_cp_compute_enable(adev, enable);
3722 }
3723
3724 static int gfx_v9_0_hw_init(void *handle)
3725 {
3726         int r;
3727         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3728
3729         if (!amdgpu_sriov_vf(adev))
3730                 gfx_v9_0_init_golden_registers(adev);
3731
3732         gfx_v9_0_constants_init(adev);
3733
3734         gfx_v9_0_init_tcp_config(adev);
3735
3736         r = adev->gfx.rlc.funcs->resume(adev);
3737         if (r)
3738                 return r;
3739
3740         r = gfx_v9_0_cp_resume(adev);
3741         if (r)
3742                 return r;
3743
3744         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
3745                 gfx_v9_4_2_set_power_brake_sequence(adev);
3746
3747         return r;
3748 }
3749
3750 static int gfx_v9_0_hw_fini(void *handle)
3751 {
3752         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3753
3754         amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
3755         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3756         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3757
3758         /* DF freeze and kcq disable will fail */
3759         if (!amdgpu_ras_intr_triggered())
3760                 /* disable KCQ to avoid CPC touch memory not valid anymore */
3761                 amdgpu_gfx_disable_kcq(adev);
3762
3763         if (amdgpu_sriov_vf(adev)) {
3764                 gfx_v9_0_cp_gfx_enable(adev, false);
3765                 /* must disable polling for SRIOV when hw finished, otherwise
3766                  * CPC engine may still keep fetching WB address which is already
3767                  * invalid after sw finished and trigger DMAR reading error in
3768                  * hypervisor side.
3769                  */
3770                 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3771                 return 0;
3772         }
3773
3774         /* Use deinitialize sequence from CAIL when unbinding device from driver,
3775          * otherwise KIQ is hanging when binding back
3776          */
3777         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3778                 mutex_lock(&adev->srbm_mutex);
3779                 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
3780                                 adev->gfx.kiq.ring.pipe,
3781                                 adev->gfx.kiq.ring.queue, 0);
3782                 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
3783                 soc15_grbm_select(adev, 0, 0, 0, 0);
3784                 mutex_unlock(&adev->srbm_mutex);
3785         }
3786
3787         gfx_v9_0_cp_enable(adev, false);
3788
3789         /* Skip stopping RLC with A+A reset or when RLC controls GFX clock */
3790         if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||
3791             (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(9, 4, 2))) {
3792                 dev_dbg(adev->dev, "Skipping RLC halt\n");
3793                 return 0;
3794         }
3795
3796         adev->gfx.rlc.funcs->stop(adev);
3797         return 0;
3798 }
3799
3800 static int gfx_v9_0_suspend(void *handle)
3801 {
3802         return gfx_v9_0_hw_fini(handle);
3803 }
3804
3805 static int gfx_v9_0_resume(void *handle)
3806 {
3807         return gfx_v9_0_hw_init(handle);
3808 }
3809
3810 static bool gfx_v9_0_is_idle(void *handle)
3811 {
3812         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3813
3814         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
3815                                 GRBM_STATUS, GUI_ACTIVE))
3816                 return false;
3817         else
3818                 return true;
3819 }
3820
3821 static int gfx_v9_0_wait_for_idle(void *handle)
3822 {
3823         unsigned i;
3824         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3825
3826         for (i = 0; i < adev->usec_timeout; i++) {
3827                 if (gfx_v9_0_is_idle(handle))
3828                         return 0;
3829                 udelay(1);
3830         }
3831         return -ETIMEDOUT;
3832 }
3833
3834 static int gfx_v9_0_soft_reset(void *handle)
3835 {
3836         u32 grbm_soft_reset = 0;
3837         u32 tmp;
3838         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3839
3840         /* GRBM_STATUS */
3841         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
3842         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
3843                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
3844                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
3845                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
3846                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
3847                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
3848                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3849                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3850                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3851                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
3852         }
3853
3854         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
3855                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3856                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3857         }
3858
3859         /* GRBM_STATUS2 */
3860         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
3861         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
3862                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3863                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3864
3865
3866         if (grbm_soft_reset) {
3867                 /* stop the rlc */
3868                 adev->gfx.rlc.funcs->stop(adev);
3869
3870                 if (adev->gfx.num_gfx_rings)
3871                         /* Disable GFX parsing/prefetching */
3872                         gfx_v9_0_cp_gfx_enable(adev, false);
3873
3874                 /* Disable MEC parsing/prefetching */
3875                 gfx_v9_0_cp_compute_enable(adev, false);
3876
3877                 if (grbm_soft_reset) {
3878                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3879                         tmp |= grbm_soft_reset;
3880                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3881                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3882                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3883
3884                         udelay(50);
3885
3886                         tmp &= ~grbm_soft_reset;
3887                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3888                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3889                 }
3890
3891                 /* Wait a little for things to settle down */
3892                 udelay(50);
3893         }
3894         return 0;
3895 }
3896
3897 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
3898 {
3899         signed long r, cnt = 0;
3900         unsigned long flags;
3901         uint32_t seq, reg_val_offs = 0;
3902         uint64_t value = 0;
3903         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
3904         struct amdgpu_ring *ring = &kiq->ring;
3905
3906         BUG_ON(!ring->funcs->emit_rreg);
3907
3908         spin_lock_irqsave(&kiq->ring_lock, flags);
3909         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
3910                 pr_err("critical bug! too many kiq readers\n");
3911                 goto failed_unlock;
3912         }
3913         amdgpu_ring_alloc(ring, 32);
3914         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
3915         amdgpu_ring_write(ring, 9 |     /* src: register*/
3916                                 (5 << 8) |      /* dst: memory */
3917                                 (1 << 16) |     /* count sel */
3918                                 (1 << 20));     /* write confirm */
3919         amdgpu_ring_write(ring, 0);
3920         amdgpu_ring_write(ring, 0);
3921         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
3922                                 reg_val_offs * 4));
3923         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
3924                                 reg_val_offs * 4));
3925         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
3926         if (r)
3927                 goto failed_undo;
3928
3929         amdgpu_ring_commit(ring);
3930         spin_unlock_irqrestore(&kiq->ring_lock, flags);
3931
3932         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
3933
3934         /* don't wait anymore for gpu reset case because this way may
3935          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
3936          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
3937          * never return if we keep waiting in virt_kiq_rreg, which cause
3938          * gpu_recover() hang there.
3939          *
3940          * also don't wait anymore for IRQ context
3941          * */
3942         if (r < 1 && (amdgpu_in_reset(adev)))
3943                 goto failed_kiq_read;
3944
3945         might_sleep();
3946         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
3947                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
3948                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
3949         }
3950
3951         if (cnt > MAX_KIQ_REG_TRY)
3952                 goto failed_kiq_read;
3953
3954         mb();
3955         value = (uint64_t)adev->wb.wb[reg_val_offs] |
3956                 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
3957         amdgpu_device_wb_free(adev, reg_val_offs);
3958         return value;
3959
3960 failed_undo:
3961         amdgpu_ring_undo(ring);
3962 failed_unlock:
3963         spin_unlock_irqrestore(&kiq->ring_lock, flags);
3964 failed_kiq_read:
3965         if (reg_val_offs)
3966                 amdgpu_device_wb_free(adev, reg_val_offs);
3967         pr_err("failed to read gpu clock\n");
3968         return ~0;
3969 }
3970
3971 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
3972 {
3973         uint64_t clock, clock_lo, clock_hi, hi_check;
3974
3975         switch (adev->ip_versions[GC_HWIP][0]) {
3976         case IP_VERSION(9, 3, 0):
3977                 preempt_disable();
3978                 clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
3979                 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
3980                 hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
3981                 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
3982                  * roughly every 42 seconds.
3983                  */
3984                 if (hi_check != clock_hi) {
3985                         clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
3986                         clock_hi = hi_check;
3987                 }
3988                 preempt_enable();
3989                 clock = clock_lo | (clock_hi << 32ULL);
3990                 break;
3991         default:
3992                 amdgpu_gfx_off_ctrl(adev, false);
3993                 mutex_lock(&adev->gfx.gpu_clock_mutex);
3994                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 0, 1) && amdgpu_sriov_runtime(adev)) {
3995                         clock = gfx_v9_0_kiq_read_clock(adev);
3996                 } else {
3997                         WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
3998                         clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
3999                                 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4000                 }
4001                 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4002                 amdgpu_gfx_off_ctrl(adev, true);
4003                 break;
4004         }
4005         return clock;
4006 }
4007
4008 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4009                                           uint32_t vmid,
4010                                           uint32_t gds_base, uint32_t gds_size,
4011                                           uint32_t gws_base, uint32_t gws_size,
4012                                           uint32_t oa_base, uint32_t oa_size)
4013 {
4014         struct amdgpu_device *adev = ring->adev;
4015
4016         /* GDS Base */
4017         gfx_v9_0_write_data_to_reg(ring, 0, false,
4018                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4019                                    gds_base);
4020
4021         /* GDS Size */
4022         gfx_v9_0_write_data_to_reg(ring, 0, false,
4023                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4024                                    gds_size);
4025
4026         /* GWS */
4027         gfx_v9_0_write_data_to_reg(ring, 0, false,
4028                                    SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4029                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4030
4031         /* OA */
4032         gfx_v9_0_write_data_to_reg(ring, 0, false,
4033                                    SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4034                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
4035 }
4036
4037 static const u32 vgpr_init_compute_shader[] =
4038 {
4039         0xb07c0000, 0xbe8000ff,
4040         0x000000f8, 0xbf110800,
4041         0x7e000280, 0x7e020280,
4042         0x7e040280, 0x7e060280,
4043         0x7e080280, 0x7e0a0280,
4044         0x7e0c0280, 0x7e0e0280,
4045         0x80808800, 0xbe803200,
4046         0xbf84fff5, 0xbf9c0000,
4047         0xd28c0001, 0x0001007f,
4048         0xd28d0001, 0x0002027e,
4049         0x10020288, 0xb8810904,
4050         0xb7814000, 0xd1196a01,
4051         0x00000301, 0xbe800087,
4052         0xbefc00c1, 0xd89c4000,
4053         0x00020201, 0xd89cc080,
4054         0x00040401, 0x320202ff,
4055         0x00000800, 0x80808100,
4056         0xbf84fff8, 0x7e020280,
4057         0xbf810000, 0x00000000,
4058 };
4059
4060 static const u32 sgpr_init_compute_shader[] =
4061 {
4062         0xb07c0000, 0xbe8000ff,
4063         0x0000005f, 0xbee50080,
4064         0xbe812c65, 0xbe822c65,
4065         0xbe832c65, 0xbe842c65,
4066         0xbe852c65, 0xb77c0005,
4067         0x80808500, 0xbf84fff8,
4068         0xbe800080, 0xbf810000,
4069 };
4070
4071 static const u32 vgpr_init_compute_shader_arcturus[] = {
4072         0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4073         0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4074         0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4075         0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4076         0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4077         0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4078         0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4079         0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4080         0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4081         0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4082         0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4083         0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4084         0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4085         0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4086         0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4087         0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4088         0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4089         0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4090         0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4091         0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4092         0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4093         0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4094         0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4095         0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4096         0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4097         0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4098         0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4099         0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4100         0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4101         0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4102         0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4103         0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4104         0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4105         0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4106         0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4107         0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4108         0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4109         0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4110         0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4111         0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4112         0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4113         0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4114         0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4115         0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4116         0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4117         0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4118         0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4119         0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4120         0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4121         0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4122         0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4123         0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4124         0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4125         0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4126         0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4127         0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4128         0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4129         0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4130         0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4131         0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4132         0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4133         0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4134         0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4135         0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4136         0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4137         0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4138         0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4139         0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4140         0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4141         0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4142         0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4143         0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4144         0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4145         0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4146         0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4147         0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4148         0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4149         0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4150         0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4151         0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4152         0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4153         0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4154         0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4155         0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4156         0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4157         0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4158         0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4159         0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4160         0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4161         0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4162         0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4163         0xbf84fff8, 0xbf810000,
4164 };
4165
4166 /* When below register arrays changed, please update gpr_reg_size,
4167   and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4168   to cover all gfx9 ASICs */
4169 static const struct soc15_reg_entry vgpr_init_regs[] = {
4170    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4171    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4172    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4173    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4174    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4175    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4176    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4177    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4178    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4179    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4180    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4181    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4182    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4183    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4184 };
4185
4186 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4187    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4188    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4189    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4190    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4191    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4192    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4193    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4194    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4195    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4196    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4197    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4198    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4199    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4200    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4201 };
4202
4203 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4204    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4205    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4206    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4207    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4208    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4209    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4210    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4211    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4212    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4213    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4214    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4215    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4216    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4217    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4218 };
4219
4220 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4221    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4222    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4223    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4224    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4225    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4226    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4227    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4228    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4229    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4230    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4231    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4232    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4233    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4234    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4235 };
4236
4237 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4238    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4239    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4240    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4241    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4242    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4243    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4244    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4245    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4246    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4247    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4248    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4249    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4250    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4251    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4252    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4253    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4254    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4255    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4256    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4257    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4258    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4259    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4260    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4261    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4262    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4263    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4264    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4265    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4266    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4267    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4268    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4269    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4270    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4271 };
4272
4273 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4274 {
4275         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4276         int i, r;
4277
4278         /* only support when RAS is enabled */
4279         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4280                 return 0;
4281
4282         r = amdgpu_ring_alloc(ring, 7);
4283         if (r) {
4284                 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4285                         ring->name, r);
4286                 return r;
4287         }
4288
4289         WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4290         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4291
4292         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4293         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4294                                 PACKET3_DMA_DATA_DST_SEL(1) |
4295                                 PACKET3_DMA_DATA_SRC_SEL(2) |
4296                                 PACKET3_DMA_DATA_ENGINE(0)));
4297         amdgpu_ring_write(ring, 0);
4298         amdgpu_ring_write(ring, 0);
4299         amdgpu_ring_write(ring, 0);
4300         amdgpu_ring_write(ring, 0);
4301         amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4302                                 adev->gds.gds_size);
4303
4304         amdgpu_ring_commit(ring);
4305
4306         for (i = 0; i < adev->usec_timeout; i++) {
4307                 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4308                         break;
4309                 udelay(1);
4310         }
4311
4312         if (i >= adev->usec_timeout)
4313                 r = -ETIMEDOUT;
4314
4315         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4316
4317         return r;
4318 }
4319
4320 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4321 {
4322         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4323         struct amdgpu_ib ib;
4324         struct dma_fence *f = NULL;
4325         int r, i;
4326         unsigned total_size, vgpr_offset, sgpr_offset;
4327         u64 gpu_addr;
4328
4329         int compute_dim_x = adev->gfx.config.max_shader_engines *
4330                                                 adev->gfx.config.max_cu_per_sh *
4331                                                 adev->gfx.config.max_sh_per_se;
4332         int sgpr_work_group_size = 5;
4333         int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4334         int vgpr_init_shader_size;
4335         const u32 *vgpr_init_shader_ptr;
4336         const struct soc15_reg_entry *vgpr_init_regs_ptr;
4337
4338         /* only support when RAS is enabled */
4339         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4340                 return 0;
4341
4342         /* bail if the compute ring is not ready */
4343         if (!ring->sched.ready)
4344                 return 0;
4345
4346         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) {
4347                 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4348                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4349                 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4350         } else {
4351                 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4352                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4353                 vgpr_init_regs_ptr = vgpr_init_regs;
4354         }
4355
4356         total_size =
4357                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4358         total_size +=
4359                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4360         total_size +=
4361                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4362         total_size = ALIGN(total_size, 256);
4363         vgpr_offset = total_size;
4364         total_size += ALIGN(vgpr_init_shader_size, 256);
4365         sgpr_offset = total_size;
4366         total_size += sizeof(sgpr_init_compute_shader);
4367
4368         /* allocate an indirect buffer to put the commands in */
4369         memset(&ib, 0, sizeof(ib));
4370         r = amdgpu_ib_get(adev, NULL, total_size,
4371                                         AMDGPU_IB_POOL_DIRECT, &ib);
4372         if (r) {
4373                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4374                 return r;
4375         }
4376
4377         /* load the compute shaders */
4378         for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4379                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4380
4381         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4382                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4383
4384         /* init the ib length to 0 */
4385         ib.length_dw = 0;
4386
4387         /* VGPR */
4388         /* write the register state for the compute dispatch */
4389         for (i = 0; i < gpr_reg_size; i++) {
4390                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4391                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4392                                                                 - PACKET3_SET_SH_REG_START;
4393                 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4394         }
4395         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4396         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4397         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4398         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4399                                                         - PACKET3_SET_SH_REG_START;
4400         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4401         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4402
4403         /* write dispatch packet */
4404         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4405         ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4406         ib.ptr[ib.length_dw++] = 1; /* y */
4407         ib.ptr[ib.length_dw++] = 1; /* z */
4408         ib.ptr[ib.length_dw++] =
4409                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4410
4411         /* write CS partial flush packet */
4412         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4413         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4414
4415         /* SGPR1 */
4416         /* write the register state for the compute dispatch */
4417         for (i = 0; i < gpr_reg_size; i++) {
4418                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4419                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4420                                                                 - PACKET3_SET_SH_REG_START;
4421                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4422         }
4423         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4424         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4425         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4426         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4427                                                         - PACKET3_SET_SH_REG_START;
4428         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4429         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4430
4431         /* write dispatch packet */
4432         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4433         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4434         ib.ptr[ib.length_dw++] = 1; /* y */
4435         ib.ptr[ib.length_dw++] = 1; /* z */
4436         ib.ptr[ib.length_dw++] =
4437                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4438
4439         /* write CS partial flush packet */
4440         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4441         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4442
4443         /* SGPR2 */
4444         /* write the register state for the compute dispatch */
4445         for (i = 0; i < gpr_reg_size; i++) {
4446                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4447                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4448                                                                 - PACKET3_SET_SH_REG_START;
4449                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4450         }
4451         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4452         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4453         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4454         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4455                                                         - PACKET3_SET_SH_REG_START;
4456         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4457         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4458
4459         /* write dispatch packet */
4460         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4461         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4462         ib.ptr[ib.length_dw++] = 1; /* y */
4463         ib.ptr[ib.length_dw++] = 1; /* z */
4464         ib.ptr[ib.length_dw++] =
4465                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4466
4467         /* write CS partial flush packet */
4468         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4469         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4470
4471         /* shedule the ib on the ring */
4472         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4473         if (r) {
4474                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4475                 goto fail;
4476         }
4477
4478         /* wait for the GPU to finish processing the IB */
4479         r = dma_fence_wait(f, false);
4480         if (r) {
4481                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4482                 goto fail;
4483         }
4484
4485 fail:
4486         amdgpu_ib_free(adev, &ib, NULL);
4487         dma_fence_put(f);
4488
4489         return r;
4490 }
4491
4492 static int gfx_v9_0_early_init(void *handle)
4493 {
4494         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4495
4496         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
4497
4498         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) ||
4499             adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
4500                 adev->gfx.num_gfx_rings = 0;
4501         else
4502                 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4503         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4504                                           AMDGPU_MAX_COMPUTE_RINGS);
4505         gfx_v9_0_set_kiq_pm4_funcs(adev);
4506         gfx_v9_0_set_ring_funcs(adev);
4507         gfx_v9_0_set_irq_funcs(adev);
4508         gfx_v9_0_set_gds_init(adev);
4509         gfx_v9_0_set_rlc_funcs(adev);
4510
4511         /* init rlcg reg access ctrl */
4512         gfx_v9_0_init_rlcg_reg_access_ctrl(adev);
4513
4514         return gfx_v9_0_init_microcode(adev);
4515 }
4516
4517 static int gfx_v9_0_ecc_late_init(void *handle)
4518 {
4519         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4520         int r;
4521
4522         /*
4523          * Temp workaround to fix the issue that CP firmware fails to
4524          * update read pointer when CPDMA is writing clearing operation
4525          * to GDS in suspend/resume sequence on several cards. So just
4526          * limit this operation in cold boot sequence.
4527          */
4528         if ((!adev->in_suspend) &&
4529             (adev->gds.gds_size)) {
4530                 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4531                 if (r)
4532                         return r;
4533         }
4534
4535         /* requires IBs so do in late init after IB pool is initialized */
4536         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
4537                 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4538         else
4539                 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4540
4541         if (r)
4542                 return r;
4543
4544         if (adev->gfx.ras &&
4545             adev->gfx.ras->enable_watchdog_timer)
4546                 adev->gfx.ras->enable_watchdog_timer(adev);
4547
4548         return 0;
4549 }
4550
4551 static int gfx_v9_0_late_init(void *handle)
4552 {
4553         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4554         int r;
4555
4556         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4557         if (r)
4558                 return r;
4559
4560         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4561         if (r)
4562                 return r;
4563
4564         r = gfx_v9_0_ecc_late_init(handle);
4565         if (r)
4566                 return r;
4567
4568         return 0;
4569 }
4570
4571 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4572 {
4573         uint32_t rlc_setting;
4574
4575         /* if RLC is not enabled, do nothing */
4576         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4577         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4578                 return false;
4579
4580         return true;
4581 }
4582
4583 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4584 {
4585         uint32_t data;
4586         unsigned i;
4587
4588         data = RLC_SAFE_MODE__CMD_MASK;
4589         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4590         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4591
4592         /* wait for RLC_SAFE_MODE */
4593         for (i = 0; i < adev->usec_timeout; i++) {
4594                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4595                         break;
4596                 udelay(1);
4597         }
4598 }
4599
4600 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4601 {
4602         uint32_t data;
4603
4604         data = RLC_SAFE_MODE__CMD_MASK;
4605         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4606 }
4607
4608 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4609                                                 bool enable)
4610 {
4611         amdgpu_gfx_rlc_enter_safe_mode(adev);
4612
4613         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4614                 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4615                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4616                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4617         } else {
4618                 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4619                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4620                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4621         }
4622
4623         amdgpu_gfx_rlc_exit_safe_mode(adev);
4624 }
4625
4626 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4627                                                 bool enable)
4628 {
4629         /* TODO: double check if we need to perform under safe mode */
4630         /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4631
4632         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4633                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4634         else
4635                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4636
4637         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4638                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4639         else
4640                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4641
4642         /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4643 }
4644
4645 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4646                                                       bool enable)
4647 {
4648         uint32_t data, def;
4649
4650         amdgpu_gfx_rlc_enter_safe_mode(adev);
4651
4652         /* It is disabled by HW by default */
4653         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4654                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4655                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4656
4657                 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 2, 1))
4658                         data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4659
4660                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4661                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4662                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4663
4664                 /* only for Vega10 & Raven1 */
4665                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4666
4667                 if (def != data)
4668                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4669
4670                 /* MGLS is a global flag to control all MGLS in GFX */
4671                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4672                         /* 2 - RLC memory Light sleep */
4673                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4674                                 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4675                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4676                                 if (def != data)
4677                                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4678                         }
4679                         /* 3 - CP memory Light sleep */
4680                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4681                                 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4682                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4683                                 if (def != data)
4684                                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4685                         }
4686                 }
4687         } else {
4688                 /* 1 - MGCG_OVERRIDE */
4689                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4690
4691                 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 2, 1))
4692                         data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4693
4694                 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4695                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4696                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4697                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4698
4699                 if (def != data)
4700                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4701
4702                 /* 2 - disable MGLS in RLC */
4703                 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4704                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4705                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4706                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4707                 }
4708
4709                 /* 3 - disable MGLS in CP */
4710                 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4711                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4712                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4713                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4714                 }
4715         }
4716
4717         amdgpu_gfx_rlc_exit_safe_mode(adev);
4718 }
4719
4720 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4721                                            bool enable)
4722 {
4723         uint32_t data, def;
4724
4725         if (!adev->gfx.num_gfx_rings)
4726                 return;
4727
4728         amdgpu_gfx_rlc_enter_safe_mode(adev);
4729
4730         /* Enable 3D CGCG/CGLS */
4731         if (enable) {
4732                 /* write cmd to clear cgcg/cgls ov */
4733                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4734                 /* unset CGCG override */
4735                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4736                 /* update CGCG and CGLS override bits */
4737                 if (def != data)
4738                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4739
4740                 /* enable 3Dcgcg FSM(0x0000363f) */
4741                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4742
4743                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
4744                         data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4745                                 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4746                 else
4747                         data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
4748
4749                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4750                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4751                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4752                 if (def != data)
4753                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4754
4755                 /* set IDLE_POLL_COUNT(0x00900100) */
4756                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4757                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4758                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4759                 if (def != data)
4760                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4761         } else {
4762                 /* Disable CGCG/CGLS */
4763                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4764                 /* disable cgcg, cgls should be disabled */
4765                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4766                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4767                 /* disable cgcg and cgls in FSM */
4768                 if (def != data)
4769                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4770         }
4771
4772         amdgpu_gfx_rlc_exit_safe_mode(adev);
4773 }
4774
4775 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4776                                                       bool enable)
4777 {
4778         uint32_t def, data;
4779
4780         amdgpu_gfx_rlc_enter_safe_mode(adev);
4781
4782         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4783                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4784                 /* unset CGCG override */
4785                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4786                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4787                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4788                 else
4789                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4790                 /* update CGCG and CGLS override bits */
4791                 if (def != data)
4792                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4793
4794                 /* enable cgcg FSM(0x0000363F) */
4795                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4796
4797                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1))
4798                         data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4799                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4800                 else
4801                         data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4802                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4803                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4804                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4805                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4806                 if (def != data)
4807                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4808
4809                 /* set IDLE_POLL_COUNT(0x00900100) */
4810                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4811                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4812                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4813                 if (def != data)
4814                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4815         } else {
4816                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4817                 /* reset CGCG/CGLS bits */
4818                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
4819                 /* disable cgcg and cgls in FSM */
4820                 if (def != data)
4821                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4822         }
4823
4824         amdgpu_gfx_rlc_exit_safe_mode(adev);
4825 }
4826
4827 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4828                                             bool enable)
4829 {
4830         if (enable) {
4831                 /* CGCG/CGLS should be enabled after MGCG/MGLS
4832                  * ===  MGCG + MGLS ===
4833                  */
4834                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4835                 /* ===  CGCG /CGLS for GFX 3D Only === */
4836                 gfx_v9_0_update_3d_clock_gating(adev, enable);
4837                 /* ===  CGCG + CGLS === */
4838                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4839         } else {
4840                 /* CGCG/CGLS should be disabled before MGCG/MGLS
4841                  * ===  CGCG + CGLS ===
4842                  */
4843                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4844                 /* ===  CGCG /CGLS for GFX 3D Only === */
4845                 gfx_v9_0_update_3d_clock_gating(adev, enable);
4846                 /* ===  MGCG + MGLS === */
4847                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4848         }
4849         return 0;
4850 }
4851
4852 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
4853 {
4854         u32 reg, data;
4855
4856         amdgpu_gfx_off_ctrl(adev, false);
4857
4858         reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
4859         if (amdgpu_sriov_is_pp_one_vf(adev))
4860                 data = RREG32_NO_KIQ(reg);
4861         else
4862                 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);
4863
4864         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
4865         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
4866
4867         if (amdgpu_sriov_is_pp_one_vf(adev))
4868                 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
4869         else
4870                 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
4871
4872         amdgpu_gfx_off_ctrl(adev, true);
4873 }
4874
4875 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
4876                                         uint32_t offset,
4877                                         struct soc15_reg_rlcg *entries, int arr_size)
4878 {
4879         int i;
4880         uint32_t reg;
4881
4882         if (!entries)
4883                 return false;
4884
4885         for (i = 0; i < arr_size; i++) {
4886                 const struct soc15_reg_rlcg *entry;
4887
4888                 entry = &entries[i];
4889                 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
4890                 if (offset == reg)
4891                         return true;
4892         }
4893
4894         return false;
4895 }
4896
4897 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
4898 {
4899         return gfx_v9_0_check_rlcg_range(adev, offset,
4900                                         (void *)rlcg_access_gc_9_0,
4901                                         ARRAY_SIZE(rlcg_access_gc_9_0));
4902 }
4903
4904 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
4905         .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
4906         .set_safe_mode = gfx_v9_0_set_safe_mode,
4907         .unset_safe_mode = gfx_v9_0_unset_safe_mode,
4908         .init = gfx_v9_0_rlc_init,
4909         .get_csb_size = gfx_v9_0_get_csb_size,
4910         .get_csb_buffer = gfx_v9_0_get_csb_buffer,
4911         .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
4912         .resume = gfx_v9_0_rlc_resume,
4913         .stop = gfx_v9_0_rlc_stop,
4914         .reset = gfx_v9_0_rlc_reset,
4915         .start = gfx_v9_0_rlc_start,
4916         .update_spm_vmid = gfx_v9_0_update_spm_vmid,
4917         .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
4918 };
4919
4920 static int gfx_v9_0_set_powergating_state(void *handle,
4921                                           enum amd_powergating_state state)
4922 {
4923         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4924         bool enable = (state == AMD_PG_STATE_GATE);
4925
4926         switch (adev->ip_versions[GC_HWIP][0]) {
4927         case IP_VERSION(9, 2, 2):
4928         case IP_VERSION(9, 1, 0):
4929         case IP_VERSION(9, 3, 0):
4930                 if (!enable)
4931                         amdgpu_gfx_off_ctrl(adev, false);
4932
4933                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
4934                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
4935                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
4936                 } else {
4937                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
4938                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
4939                 }
4940
4941                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
4942                         gfx_v9_0_enable_cp_power_gating(adev, true);
4943                 else
4944                         gfx_v9_0_enable_cp_power_gating(adev, false);
4945
4946                 /* update gfx cgpg state */
4947                 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
4948
4949                 /* update mgcg state */
4950                 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
4951
4952                 if (enable)
4953                         amdgpu_gfx_off_ctrl(adev, true);
4954                 break;
4955         case IP_VERSION(9, 2, 1):
4956                 amdgpu_gfx_off_ctrl(adev, enable);
4957                 break;
4958         default:
4959                 break;
4960         }
4961
4962         return 0;
4963 }
4964
4965 static int gfx_v9_0_set_clockgating_state(void *handle,
4966                                           enum amd_clockgating_state state)
4967 {
4968         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4969
4970         if (amdgpu_sriov_vf(adev))
4971                 return 0;
4972
4973         switch (adev->ip_versions[GC_HWIP][0]) {
4974         case IP_VERSION(9, 0, 1):
4975         case IP_VERSION(9, 2, 1):
4976         case IP_VERSION(9, 4, 0):
4977         case IP_VERSION(9, 2, 2):
4978         case IP_VERSION(9, 1, 0):
4979         case IP_VERSION(9, 4, 1):
4980         case IP_VERSION(9, 3, 0):
4981         case IP_VERSION(9, 4, 2):
4982                 gfx_v9_0_update_gfx_clock_gating(adev,
4983                                                  state == AMD_CG_STATE_GATE);
4984                 break;
4985         default:
4986                 break;
4987         }
4988         return 0;
4989 }
4990
4991 static void gfx_v9_0_get_clockgating_state(void *handle, u64 *flags)
4992 {
4993         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4994         int data;
4995
4996         if (amdgpu_sriov_vf(adev))
4997                 *flags = 0;
4998
4999         /* AMD_CG_SUPPORT_GFX_MGCG */
5000         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5001         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5002                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5003
5004         /* AMD_CG_SUPPORT_GFX_CGCG */
5005         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5006         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5007                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5008
5009         /* AMD_CG_SUPPORT_GFX_CGLS */
5010         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5011                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5012
5013         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5014         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5015         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5016                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5017
5018         /* AMD_CG_SUPPORT_GFX_CP_LS */
5019         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5020         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5021                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5022
5023         if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) {
5024                 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5025                 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5026                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5027                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5028
5029                 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5030                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5031                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5032         }
5033 }
5034
5035 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5036 {
5037         return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/
5038 }
5039
5040 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5041 {
5042         struct amdgpu_device *adev = ring->adev;
5043         u64 wptr;
5044
5045         /* XXX check if swapping is necessary on BE */
5046         if (ring->use_doorbell) {
5047                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5048         } else {
5049                 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5050                 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5051         }
5052
5053         return wptr;
5054 }
5055
5056 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5057 {
5058         struct amdgpu_device *adev = ring->adev;
5059
5060         if (ring->use_doorbell) {
5061                 /* XXX check if swapping is necessary on BE */
5062                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5063                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5064         } else {
5065                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5066                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5067         }
5068 }
5069
5070 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5071 {
5072         struct amdgpu_device *adev = ring->adev;
5073         u32 ref_and_mask, reg_mem_engine;
5074         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5075
5076         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5077                 switch (ring->me) {
5078                 case 1:
5079                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5080                         break;
5081                 case 2:
5082                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5083                         break;
5084                 default:
5085                         return;
5086                 }
5087                 reg_mem_engine = 0;
5088         } else {
5089                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5090                 reg_mem_engine = 1; /* pfp */
5091         }
5092
5093         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5094                               adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5095                               adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5096                               ref_and_mask, ref_and_mask, 0x20);
5097 }
5098
5099 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5100                                         struct amdgpu_job *job,
5101                                         struct amdgpu_ib *ib,
5102                                         uint32_t flags)
5103 {
5104         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5105         u32 header, control = 0;
5106
5107         if (ib->flags & AMDGPU_IB_FLAG_CE)
5108                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5109         else
5110                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5111
5112         control |= ib->length_dw | (vmid << 24);
5113
5114         if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
5115                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5116
5117                 if (flags & AMDGPU_IB_PREEMPTED)
5118                         control |= INDIRECT_BUFFER_PRE_RESUME(1);
5119
5120                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5121                         gfx_v9_0_ring_emit_de_meta(ring,
5122                                                    (!amdgpu_sriov_vf(ring->adev) &&
5123                                                    flags & AMDGPU_IB_PREEMPTED) ?
5124                                                    true : false);
5125         }
5126
5127         amdgpu_ring_write(ring, header);
5128         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5129         amdgpu_ring_write(ring,
5130 #ifdef __BIG_ENDIAN
5131                 (2 << 0) |
5132 #endif
5133                 lower_32_bits(ib->gpu_addr));
5134         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5135         amdgpu_ring_write(ring, control);
5136 }
5137
5138 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5139                                           struct amdgpu_job *job,
5140                                           struct amdgpu_ib *ib,
5141                                           uint32_t flags)
5142 {
5143         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5144         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5145
5146         /* Currently, there is a high possibility to get wave ID mismatch
5147          * between ME and GDS, leading to a hw deadlock, because ME generates
5148          * different wave IDs than the GDS expects. This situation happens
5149          * randomly when at least 5 compute pipes use GDS ordered append.
5150          * The wave IDs generated by ME are also wrong after suspend/resume.
5151          * Those are probably bugs somewhere else in the kernel driver.
5152          *
5153          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5154          * GDS to 0 for this ring (me/pipe).
5155          */
5156         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5157                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5158                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5159                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5160         }
5161
5162         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5163         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5164         amdgpu_ring_write(ring,
5165 #ifdef __BIG_ENDIAN
5166                                 (2 << 0) |
5167 #endif
5168                                 lower_32_bits(ib->gpu_addr));
5169         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5170         amdgpu_ring_write(ring, control);
5171 }
5172
5173 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5174                                      u64 seq, unsigned flags)
5175 {
5176         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5177         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5178         bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5179         bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;
5180         uint32_t dw2 = 0;
5181
5182         /* RELEASE_MEM - flush caches, send int */
5183         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5184
5185         if (writeback) {
5186                 dw2 = EOP_TC_NC_ACTION_EN;
5187         } else {
5188                 dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN |
5189                                 EOP_TC_MD_ACTION_EN;
5190         }
5191         dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5192                                 EVENT_INDEX(5);
5193         if (exec)
5194                 dw2 |= EOP_EXEC;
5195
5196         amdgpu_ring_write(ring, dw2);
5197         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5198
5199         /*
5200          * the address should be Qword aligned if 64bit write, Dword
5201          * aligned if only send 32bit data low (discard data high)
5202          */
5203         if (write64bit)
5204                 BUG_ON(addr & 0x7);
5205         else
5206                 BUG_ON(addr & 0x3);
5207         amdgpu_ring_write(ring, lower_32_bits(addr));
5208         amdgpu_ring_write(ring, upper_32_bits(addr));
5209         amdgpu_ring_write(ring, lower_32_bits(seq));
5210         amdgpu_ring_write(ring, upper_32_bits(seq));
5211         amdgpu_ring_write(ring, 0);
5212 }
5213
5214 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5215 {
5216         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5217         uint32_t seq = ring->fence_drv.sync_seq;
5218         uint64_t addr = ring->fence_drv.gpu_addr;
5219
5220         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5221                               lower_32_bits(addr), upper_32_bits(addr),
5222                               seq, 0xffffffff, 4);
5223 }
5224
5225 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5226                                         unsigned vmid, uint64_t pd_addr)
5227 {
5228         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5229
5230         /* compute doesn't have PFP */
5231         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5232                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5233                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5234                 amdgpu_ring_write(ring, 0x0);
5235         }
5236 }
5237
5238 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5239 {
5240         return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */
5241 }
5242
5243 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5244 {
5245         u64 wptr;
5246
5247         /* XXX check if swapping is necessary on BE */
5248         if (ring->use_doorbell)
5249                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5250         else
5251                 BUG();
5252         return wptr;
5253 }
5254
5255 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5256 {
5257         struct amdgpu_device *adev = ring->adev;
5258
5259         /* XXX check if swapping is necessary on BE */
5260         if (ring->use_doorbell) {
5261                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5262                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5263         } else{
5264                 BUG(); /* only DOORBELL method supported on gfx9 now */
5265         }
5266 }
5267
5268 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5269                                          u64 seq, unsigned int flags)
5270 {
5271         struct amdgpu_device *adev = ring->adev;
5272
5273         /* we only allocate 32bit for each seq wb address */
5274         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5275
5276         /* write fence seq to the "addr" */
5277         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5278         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5279                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5280         amdgpu_ring_write(ring, lower_32_bits(addr));
5281         amdgpu_ring_write(ring, upper_32_bits(addr));
5282         amdgpu_ring_write(ring, lower_32_bits(seq));
5283
5284         if (flags & AMDGPU_FENCE_FLAG_INT) {
5285                 /* set register to trigger INT */
5286                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5287                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5288                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5289                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5290                 amdgpu_ring_write(ring, 0);
5291                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5292         }
5293 }
5294
5295 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5296 {
5297         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5298         amdgpu_ring_write(ring, 0);
5299 }
5300
5301 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
5302 {
5303         struct amdgpu_device *adev = ring->adev;
5304         struct v9_ce_ib_state ce_payload = {0};
5305         uint64_t offset, ce_payload_gpu_addr;
5306         void *ce_payload_cpu_addr;
5307         int cnt;
5308
5309         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5310
5311         if (ring->is_mes_queue) {
5312                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5313                                   gfx[0].gfx_meta_data) +
5314                         offsetof(struct v9_gfx_meta_data, ce_payload);
5315                 ce_payload_gpu_addr =
5316                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5317                 ce_payload_cpu_addr =
5318                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5319         } else {
5320                 offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5321                 ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5322                 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5323         }
5324
5325         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5326         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5327                                  WRITE_DATA_DST_SEL(8) |
5328                                  WR_CONFIRM) |
5329                                  WRITE_DATA_CACHE_POLICY(0));
5330         amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));
5331         amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));
5332
5333         if (resume)
5334                 amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr,
5335                                            sizeof(ce_payload) >> 2);
5336         else
5337                 amdgpu_ring_write_multiple(ring, (void *)&ce_payload,
5338                                            sizeof(ce_payload) >> 2);
5339 }
5340
5341 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)
5342 {
5343         int i, r = 0;
5344         struct amdgpu_device *adev = ring->adev;
5345         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
5346         struct amdgpu_ring *kiq_ring = &kiq->ring;
5347         unsigned long flags;
5348
5349         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5350                 return -EINVAL;
5351
5352         spin_lock_irqsave(&kiq->ring_lock, flags);
5353
5354         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5355                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5356                 return -ENOMEM;
5357         }
5358
5359         /* assert preemption condition */
5360         amdgpu_ring_set_preempt_cond_exec(ring, false);
5361
5362         ring->trail_seq += 1;
5363         amdgpu_ring_alloc(ring, 13);
5364         gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
5365                                  ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);
5366         /*reset the CP_VMID_PREEMPT after trailing fence*/
5367         amdgpu_ring_emit_wreg(ring,
5368                               SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT),
5369                               0x0);
5370
5371         /* assert IB preemption, emit the trailing fence */
5372         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5373                                    ring->trail_fence_gpu_addr,
5374                                    ring->trail_seq);
5375
5376         amdgpu_ring_commit(kiq_ring);
5377         spin_unlock_irqrestore(&kiq->ring_lock, flags);
5378
5379         /* poll the trailing fence */
5380         for (i = 0; i < adev->usec_timeout; i++) {
5381                 if (ring->trail_seq ==
5382                         le32_to_cpu(*ring->trail_fence_cpu_addr))
5383                         break;
5384                 udelay(1);
5385         }
5386
5387         if (i >= adev->usec_timeout) {
5388                 r = -EINVAL;
5389                 DRM_WARN("ring %d timeout to preempt ib\n", ring->idx);
5390         }
5391
5392         amdgpu_ring_commit(ring);
5393
5394         /* deassert preemption condition */
5395         amdgpu_ring_set_preempt_cond_exec(ring, true);
5396         return r;
5397 }
5398
5399 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
5400 {
5401         struct amdgpu_device *adev = ring->adev;
5402         struct v9_de_ib_state de_payload = {0};
5403         uint64_t offset, gds_addr, de_payload_gpu_addr;
5404         void *de_payload_cpu_addr;
5405         int cnt;
5406
5407         if (ring->is_mes_queue) {
5408                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5409                                   gfx[0].gfx_meta_data) +
5410                         offsetof(struct v9_gfx_meta_data, de_payload);
5411                 de_payload_gpu_addr =
5412                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5413                 de_payload_cpu_addr =
5414                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5415
5416                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5417                                   gfx[0].gds_backup) +
5418                         offsetof(struct v9_gfx_meta_data, de_payload);
5419                 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5420         } else {
5421                 offset = offsetof(struct v9_gfx_meta_data, de_payload);
5422                 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5423                 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5424
5425                 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5426                                  AMDGPU_CSA_SIZE - adev->gds.gds_size,
5427                                  PAGE_SIZE);
5428         }
5429
5430         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5431         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5432
5433         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5434         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5435         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5436                                  WRITE_DATA_DST_SEL(8) |
5437                                  WR_CONFIRM) |
5438                                  WRITE_DATA_CACHE_POLICY(0));
5439         amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5440         amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5441
5442         if (resume)
5443                 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5444                                            sizeof(de_payload) >> 2);
5445         else
5446                 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5447                                            sizeof(de_payload) >> 2);
5448 }
5449
5450 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5451                                    bool secure)
5452 {
5453         uint32_t v = secure ? FRAME_TMZ : 0;
5454
5455         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5456         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5457 }
5458
5459 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5460 {
5461         uint32_t dw2 = 0;
5462
5463         gfx_v9_0_ring_emit_ce_meta(ring,
5464                                    (!amdgpu_sriov_vf(ring->adev) &&
5465                                    flags & AMDGPU_IB_PREEMPTED) ? true : false);
5466
5467         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5468         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5469                 /* set load_global_config & load_global_uconfig */
5470                 dw2 |= 0x8001;
5471                 /* set load_cs_sh_regs */
5472                 dw2 |= 0x01000000;
5473                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5474                 dw2 |= 0x10002;
5475
5476                 /* set load_ce_ram if preamble presented */
5477                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5478                         dw2 |= 0x10000000;
5479         } else {
5480                 /* still load_ce_ram if this is the first time preamble presented
5481                  * although there is no context switch happens.
5482                  */
5483                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5484                         dw2 |= 0x10000000;
5485         }
5486
5487         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5488         amdgpu_ring_write(ring, dw2);
5489         amdgpu_ring_write(ring, 0);
5490 }
5491
5492 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5493 {
5494         unsigned ret;
5495         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5496         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5497         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5498         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5499         ret = ring->wptr & ring->buf_mask;
5500         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5501         return ret;
5502 }
5503
5504 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5505 {
5506         unsigned cur;
5507         BUG_ON(offset > ring->buf_mask);
5508         BUG_ON(ring->ring[offset] != 0x55aa55aa);
5509
5510         cur = (ring->wptr - 1) & ring->buf_mask;
5511         if (likely(cur > offset))
5512                 ring->ring[offset] = cur - offset;
5513         else
5514                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5515 }
5516
5517 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5518                                     uint32_t reg_val_offs)
5519 {
5520         struct amdgpu_device *adev = ring->adev;
5521
5522         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5523         amdgpu_ring_write(ring, 0 |     /* src: register*/
5524                                 (5 << 8) |      /* dst: memory */
5525                                 (1 << 20));     /* write confirm */
5526         amdgpu_ring_write(ring, reg);
5527         amdgpu_ring_write(ring, 0);
5528         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5529                                 reg_val_offs * 4));
5530         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5531                                 reg_val_offs * 4));
5532 }
5533
5534 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5535                                     uint32_t val)
5536 {
5537         uint32_t cmd = 0;
5538
5539         switch (ring->funcs->type) {
5540         case AMDGPU_RING_TYPE_GFX:
5541                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5542                 break;
5543         case AMDGPU_RING_TYPE_KIQ:
5544                 cmd = (1 << 16); /* no inc addr */
5545                 break;
5546         default:
5547                 cmd = WR_CONFIRM;
5548                 break;
5549         }
5550         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5551         amdgpu_ring_write(ring, cmd);
5552         amdgpu_ring_write(ring, reg);
5553         amdgpu_ring_write(ring, 0);
5554         amdgpu_ring_write(ring, val);
5555 }
5556
5557 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5558                                         uint32_t val, uint32_t mask)
5559 {
5560         gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5561 }
5562
5563 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5564                                                   uint32_t reg0, uint32_t reg1,
5565                                                   uint32_t ref, uint32_t mask)
5566 {
5567         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5568         struct amdgpu_device *adev = ring->adev;
5569         bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5570                 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5571
5572         if (fw_version_ok)
5573                 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5574                                       ref, mask, 0x20);
5575         else
5576                 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5577                                                            ref, mask);
5578 }
5579
5580 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5581 {
5582         struct amdgpu_device *adev = ring->adev;
5583         uint32_t value = 0;
5584
5585         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5586         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5587         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5588         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5589         WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5590 }
5591
5592 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5593                                                  enum amdgpu_interrupt_state state)
5594 {
5595         switch (state) {
5596         case AMDGPU_IRQ_STATE_DISABLE:
5597         case AMDGPU_IRQ_STATE_ENABLE:
5598                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5599                                TIME_STAMP_INT_ENABLE,
5600                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5601                 break;
5602         default:
5603                 break;
5604         }
5605 }
5606
5607 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5608                                                      int me, int pipe,
5609                                                      enum amdgpu_interrupt_state state)
5610 {
5611         u32 mec_int_cntl, mec_int_cntl_reg;
5612
5613         /*
5614          * amdgpu controls only the first MEC. That's why this function only
5615          * handles the setting of interrupts for this specific MEC. All other
5616          * pipes' interrupts are set by amdkfd.
5617          */
5618
5619         if (me == 1) {
5620                 switch (pipe) {
5621                 case 0:
5622                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5623                         break;
5624                 case 1:
5625                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5626                         break;
5627                 case 2:
5628                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5629                         break;
5630                 case 3:
5631                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5632                         break;
5633                 default:
5634                         DRM_DEBUG("invalid pipe %d\n", pipe);
5635                         return;
5636                 }
5637         } else {
5638                 DRM_DEBUG("invalid me %d\n", me);
5639                 return;
5640         }
5641
5642         switch (state) {
5643         case AMDGPU_IRQ_STATE_DISABLE:
5644                 mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg);
5645                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5646                                              TIME_STAMP_INT_ENABLE, 0);
5647                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5648                 break;
5649         case AMDGPU_IRQ_STATE_ENABLE:
5650                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5651                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5652                                              TIME_STAMP_INT_ENABLE, 1);
5653                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5654                 break;
5655         default:
5656                 break;
5657         }
5658 }
5659
5660 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5661                                              struct amdgpu_irq_src *source,
5662                                              unsigned type,
5663                                              enum amdgpu_interrupt_state state)
5664 {
5665         switch (state) {
5666         case AMDGPU_IRQ_STATE_DISABLE:
5667         case AMDGPU_IRQ_STATE_ENABLE:
5668                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5669                                PRIV_REG_INT_ENABLE,
5670                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5671                 break;
5672         default:
5673                 break;
5674         }
5675
5676         return 0;
5677 }
5678
5679 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5680                                               struct amdgpu_irq_src *source,
5681                                               unsigned type,
5682                                               enum amdgpu_interrupt_state state)
5683 {
5684         switch (state) {
5685         case AMDGPU_IRQ_STATE_DISABLE:
5686         case AMDGPU_IRQ_STATE_ENABLE:
5687                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5688                                PRIV_INSTR_INT_ENABLE,
5689                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5690                 break;
5691         default:
5692                 break;
5693         }
5694
5695         return 0;
5696 }
5697
5698 #define ENABLE_ECC_ON_ME_PIPE(me, pipe)                         \
5699         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5700                         CP_ECC_ERROR_INT_ENABLE, 1)
5701
5702 #define DISABLE_ECC_ON_ME_PIPE(me, pipe)                        \
5703         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5704                         CP_ECC_ERROR_INT_ENABLE, 0)
5705
5706 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5707                                               struct amdgpu_irq_src *source,
5708                                               unsigned type,
5709                                               enum amdgpu_interrupt_state state)
5710 {
5711         switch (state) {
5712         case AMDGPU_IRQ_STATE_DISABLE:
5713                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5714                                 CP_ECC_ERROR_INT_ENABLE, 0);
5715                 DISABLE_ECC_ON_ME_PIPE(1, 0);
5716                 DISABLE_ECC_ON_ME_PIPE(1, 1);
5717                 DISABLE_ECC_ON_ME_PIPE(1, 2);
5718                 DISABLE_ECC_ON_ME_PIPE(1, 3);
5719                 break;
5720
5721         case AMDGPU_IRQ_STATE_ENABLE:
5722                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5723                                 CP_ECC_ERROR_INT_ENABLE, 1);
5724                 ENABLE_ECC_ON_ME_PIPE(1, 0);
5725                 ENABLE_ECC_ON_ME_PIPE(1, 1);
5726                 ENABLE_ECC_ON_ME_PIPE(1, 2);
5727                 ENABLE_ECC_ON_ME_PIPE(1, 3);
5728                 break;
5729         default:
5730                 break;
5731         }
5732
5733         return 0;
5734 }
5735
5736
5737 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5738                                             struct amdgpu_irq_src *src,
5739                                             unsigned type,
5740                                             enum amdgpu_interrupt_state state)
5741 {
5742         switch (type) {
5743         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5744                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5745                 break;
5746         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5747                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5748                 break;
5749         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5750                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5751                 break;
5752         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5753                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5754                 break;
5755         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5756                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5757                 break;
5758         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5759                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5760                 break;
5761         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5762                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5763                 break;
5764         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5765                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5766                 break;
5767         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5768                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5769                 break;
5770         default:
5771                 break;
5772         }
5773         return 0;
5774 }
5775
5776 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5777                             struct amdgpu_irq_src *source,
5778                             struct amdgpu_iv_entry *entry)
5779 {
5780         int i;
5781         u8 me_id, pipe_id, queue_id;
5782         struct amdgpu_ring *ring;
5783
5784         DRM_DEBUG("IH: CP EOP\n");
5785         me_id = (entry->ring_id & 0x0c) >> 2;
5786         pipe_id = (entry->ring_id & 0x03) >> 0;
5787         queue_id = (entry->ring_id & 0x70) >> 4;
5788
5789         switch (me_id) {
5790         case 0:
5791                 if (adev->gfx.num_gfx_rings &&
5792                     !amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) {
5793                         /* Fence signals are handled on the software rings*/
5794                         for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
5795                                 amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]);
5796                 }
5797                 break;
5798         case 1:
5799         case 2:
5800                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5801                         ring = &adev->gfx.compute_ring[i];
5802                         /* Per-queue interrupt is supported for MEC starting from VI.
5803                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
5804                           */
5805                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5806                                 amdgpu_fence_process(ring);
5807                 }
5808                 break;
5809         }
5810         return 0;
5811 }
5812
5813 static void gfx_v9_0_fault(struct amdgpu_device *adev,
5814                            struct amdgpu_iv_entry *entry)
5815 {
5816         u8 me_id, pipe_id, queue_id;
5817         struct amdgpu_ring *ring;
5818         int i;
5819
5820         me_id = (entry->ring_id & 0x0c) >> 2;
5821         pipe_id = (entry->ring_id & 0x03) >> 0;
5822         queue_id = (entry->ring_id & 0x70) >> 4;
5823
5824         switch (me_id) {
5825         case 0:
5826                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5827                 break;
5828         case 1:
5829         case 2:
5830                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5831                         ring = &adev->gfx.compute_ring[i];
5832                         if (ring->me == me_id && ring->pipe == pipe_id &&
5833                             ring->queue == queue_id)
5834                                 drm_sched_fault(&ring->sched);
5835                 }
5836                 break;
5837         }
5838 }
5839
5840 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5841                                  struct amdgpu_irq_src *source,
5842                                  struct amdgpu_iv_entry *entry)
5843 {
5844         DRM_ERROR("Illegal register access in command stream\n");
5845         gfx_v9_0_fault(adev, entry);
5846         return 0;
5847 }
5848
5849 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5850                                   struct amdgpu_irq_src *source,
5851                                   struct amdgpu_iv_entry *entry)
5852 {
5853         DRM_ERROR("Illegal instruction in command stream\n");
5854         gfx_v9_0_fault(adev, entry);
5855         return 0;
5856 }
5857
5858
5859 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5860         { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5861           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5862           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5863         },
5864         { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5865           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5866           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5867         },
5868         { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5869           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5870           0, 0
5871         },
5872         { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5873           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5874           0, 0
5875         },
5876         { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5877           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5878           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5879         },
5880         { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5881           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5882           0, 0
5883         },
5884         { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5885           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
5886           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
5887         },
5888         { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
5889           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
5890           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
5891         },
5892         { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
5893           SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
5894           0, 0
5895         },
5896         { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
5897           SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
5898           0, 0
5899         },
5900         { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
5901           SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
5902           0, 0
5903         },
5904         { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5905           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
5906           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
5907         },
5908         { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5909           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
5910           0, 0
5911         },
5912         { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5913           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
5914           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
5915         },
5916         { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
5917           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5918           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
5919           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
5920         },
5921         { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
5922           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5923           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
5924           0, 0
5925         },
5926         { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
5927           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5928           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
5929           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
5930         },
5931         { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
5932           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5933           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
5934           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
5935         },
5936         { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
5937           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5938           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
5939           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
5940         },
5941         { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
5942           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5943           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
5944           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
5945         },
5946         { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
5947           SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
5948           0, 0
5949         },
5950         { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5951           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
5952           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
5953         },
5954         { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5955           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
5956           0, 0
5957         },
5958         { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5959           SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
5960           0, 0
5961         },
5962         { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5963           SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
5964           0, 0
5965         },
5966         { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5967           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
5968           0, 0
5969         },
5970         { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5971           SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
5972           0, 0
5973         },
5974         { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5975           SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
5976           0, 0
5977         },
5978         { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5979           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
5980           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
5981         },
5982         { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5983           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
5984           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
5985         },
5986         { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5987           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
5988           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
5989         },
5990         { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5991           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
5992           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
5993         },
5994         { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5995           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
5996           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
5997         },
5998         { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5999           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6000           0, 0
6001         },
6002         { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6003           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6004           0, 0
6005         },
6006         { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6007           SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6008           0, 0
6009         },
6010         { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6011           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6012           0, 0
6013         },
6014         { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6015           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6016           0, 0
6017         },
6018         { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6019           SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6020           0, 0
6021         },
6022         { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6023           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6024           0, 0
6025         },
6026         { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6027           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6028           0, 0
6029         },
6030         { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6031           SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6032           0, 0
6033         },
6034         { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6035           SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6036           0, 0
6037         },
6038         { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6039           SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6040           0, 0
6041         },
6042         { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6043           SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6044           0, 0
6045         },
6046         { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6047           SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6048           0, 0
6049         },
6050         { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6051           SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6052           0, 0
6053         },
6054         { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6055           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6056           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6057         },
6058         { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6059           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6060           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6061         },
6062         { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6063           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6064           0, 0
6065         },
6066         { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6067           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6068           0, 0
6069         },
6070         { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6071           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6072           0, 0
6073         },
6074         { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6075           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6076           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6077         },
6078         { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6079           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6080           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6081         },
6082         { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6083           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6084           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6085         },
6086         { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6087           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6088           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6089         },
6090         { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6091           SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6092           0, 0
6093         },
6094         { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6095           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6096           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6097         },
6098         { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6099           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6100           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6101         },
6102         { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6103           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6104           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6105         },
6106         { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6107           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6108           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6109         },
6110         { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6111           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6112           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6113         },
6114         { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6115           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6116           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6117         },
6118         { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6119           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6120           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6121         },
6122         { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6123           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6124           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6125         },
6126         { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6127           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6128           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6129         },
6130         { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6131           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6132           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6133         },
6134         { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6135           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6136           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6137         },
6138         { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6139           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6140           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6141         },
6142         { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6143           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6144           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6145         },
6146         { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6147           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6148           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6149         },
6150         { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6151           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6152           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6153         },
6154         { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6155           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6156           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6157         },
6158         { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6159           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6160           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6161         },
6162         { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6163           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6164           0, 0
6165         },
6166         { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6167           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6168           0, 0
6169         },
6170         { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6171           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6172           0, 0
6173         },
6174         { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6175           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6176           0, 0
6177         },
6178         { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6179           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6180           0, 0
6181         },
6182         { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6183           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6184           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6185         },
6186         { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6187           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6188           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6189         },
6190         { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6191           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6192           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6193         },
6194         { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6195           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6196           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6197         },
6198         { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6199           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6200           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6201         },
6202         { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6203           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6204           0, 0
6205         },
6206         { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6207           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6208           0, 0
6209         },
6210         { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6211           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6212           0, 0
6213         },
6214         { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6215           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6216           0, 0
6217         },
6218         { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6219           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6220           0, 0
6221         },
6222         { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6223           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6224           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6225         },
6226         { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6227           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6228           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6229         },
6230         { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6231           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6232           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6233         },
6234         { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6235           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6236           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6237         },
6238         { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6239           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6240           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6241         },
6242         { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6243           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6244           0, 0
6245         },
6246         { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6247           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6248           0, 0
6249         },
6250         { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6251           SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6252           0, 0
6253         },
6254         { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6255           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6256           0, 0
6257         },
6258         { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6259           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6260           0, 0
6261         },
6262         { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6263           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6264           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6265         },
6266         { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6267           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6268           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6269         },
6270         { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6271           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6272           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6273         },
6274         { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6275           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6276           0, 0
6277         },
6278         { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6279           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6280           0, 0
6281         },
6282         { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6283           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6284           0, 0
6285         },
6286         { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6287           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6288           0, 0
6289         },
6290         { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6291           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6292           0, 0
6293         },
6294         { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6295           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6296           0, 0
6297         }
6298 };
6299
6300 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6301                                      void *inject_if)
6302 {
6303         struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6304         int ret;
6305         struct ta_ras_trigger_error_input block_info = { 0 };
6306
6307         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6308                 return -EINVAL;
6309
6310         if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6311                 return -EINVAL;
6312
6313         if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6314                 return -EPERM;
6315
6316         if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6317               info->head.type)) {
6318                 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6319                         ras_gfx_subblocks[info->head.sub_block_index].name,
6320                         info->head.type);
6321                 return -EPERM;
6322         }
6323
6324         if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6325               info->head.type)) {
6326                 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6327                         ras_gfx_subblocks[info->head.sub_block_index].name,
6328                         info->head.type);
6329                 return -EPERM;
6330         }
6331
6332         block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6333         block_info.sub_block_index =
6334                 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6335         block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6336         block_info.address = info->address;
6337         block_info.value = info->value;
6338
6339         mutex_lock(&adev->grbm_idx_mutex);
6340         ret = psp_ras_trigger_error(&adev->psp, &block_info);
6341         mutex_unlock(&adev->grbm_idx_mutex);
6342
6343         return ret;
6344 }
6345
6346 static const char *vml2_mems[] = {
6347         "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6348         "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6349         "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6350         "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6351         "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6352         "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6353         "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6354         "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6355         "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6356         "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6357         "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6358         "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6359         "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6360         "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6361         "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6362         "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6363 };
6364
6365 static const char *vml2_walker_mems[] = {
6366         "UTC_VML2_CACHE_PDE0_MEM0",
6367         "UTC_VML2_CACHE_PDE0_MEM1",
6368         "UTC_VML2_CACHE_PDE1_MEM0",
6369         "UTC_VML2_CACHE_PDE1_MEM1",
6370         "UTC_VML2_CACHE_PDE2_MEM0",
6371         "UTC_VML2_CACHE_PDE2_MEM1",
6372         "UTC_VML2_RDIF_LOG_FIFO",
6373 };
6374
6375 static const char *atc_l2_cache_2m_mems[] = {
6376         "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6377         "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6378         "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6379         "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6380 };
6381
6382 static const char *atc_l2_cache_4k_mems[] = {
6383         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6384         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6385         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6386         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6387         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6388         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6389         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6390         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6391         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6392         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6393         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6394         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6395         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6396         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6397         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6398         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6399         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6400         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6401         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6402         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6403         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6404         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6405         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6406         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6407         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6408         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6409         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6410         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6411         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6412         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6413         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6414         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6415 };
6416
6417 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6418                                          struct ras_err_data *err_data)
6419 {
6420         uint32_t i, data;
6421         uint32_t sec_count, ded_count;
6422
6423         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6424         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6425         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6426         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6427         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6428         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6429         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6430         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6431
6432         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6433                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6434                 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6435
6436                 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6437                 if (sec_count) {
6438                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6439                                 "SEC %d\n", i, vml2_mems[i], sec_count);
6440                         err_data->ce_count += sec_count;
6441                 }
6442
6443                 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6444                 if (ded_count) {
6445                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6446                                 "DED %d\n", i, vml2_mems[i], ded_count);
6447                         err_data->ue_count += ded_count;
6448                 }
6449         }
6450
6451         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6452                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6453                 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6454
6455                 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6456                                                 SEC_COUNT);
6457                 if (sec_count) {
6458                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6459                                 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6460                         err_data->ce_count += sec_count;
6461                 }
6462
6463                 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6464                                                 DED_COUNT);
6465                 if (ded_count) {
6466                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6467                                 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6468                         err_data->ue_count += ded_count;
6469                 }
6470         }
6471
6472         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6473                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6474                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6475
6476                 sec_count = (data & 0x00006000L) >> 0xd;
6477                 if (sec_count) {
6478                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6479                                 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6480                                 sec_count);
6481                         err_data->ce_count += sec_count;
6482                 }
6483         }
6484
6485         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6486                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6487                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6488
6489                 sec_count = (data & 0x00006000L) >> 0xd;
6490                 if (sec_count) {
6491                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6492                                 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6493                                 sec_count);
6494                         err_data->ce_count += sec_count;
6495                 }
6496
6497                 ded_count = (data & 0x00018000L) >> 0xf;
6498                 if (ded_count) {
6499                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6500                                 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6501                                 ded_count);
6502                         err_data->ue_count += ded_count;
6503                 }
6504         }
6505
6506         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6507         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6508         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6509         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6510
6511         return 0;
6512 }
6513
6514 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6515         const struct soc15_reg_entry *reg,
6516         uint32_t se_id, uint32_t inst_id, uint32_t value,
6517         uint32_t *sec_count, uint32_t *ded_count)
6518 {
6519         uint32_t i;
6520         uint32_t sec_cnt, ded_cnt;
6521
6522         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6523                 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6524                         gfx_v9_0_ras_fields[i].seg != reg->seg ||
6525                         gfx_v9_0_ras_fields[i].inst != reg->inst)
6526                         continue;
6527
6528                 sec_cnt = (value &
6529                                 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6530                                 gfx_v9_0_ras_fields[i].sec_count_shift;
6531                 if (sec_cnt) {
6532                         dev_info(adev->dev, "GFX SubBlock %s, "
6533                                 "Instance[%d][%d], SEC %d\n",
6534                                 gfx_v9_0_ras_fields[i].name,
6535                                 se_id, inst_id,
6536                                 sec_cnt);
6537                         *sec_count += sec_cnt;
6538                 }
6539
6540                 ded_cnt = (value &
6541                                 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6542                                 gfx_v9_0_ras_fields[i].ded_count_shift;
6543                 if (ded_cnt) {
6544                         dev_info(adev->dev, "GFX SubBlock %s, "
6545                                 "Instance[%d][%d], DED %d\n",
6546                                 gfx_v9_0_ras_fields[i].name,
6547                                 se_id, inst_id,
6548                                 ded_cnt);
6549                         *ded_count += ded_cnt;
6550                 }
6551         }
6552
6553         return 0;
6554 }
6555
6556 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6557 {
6558         int i, j, k;
6559
6560         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6561                 return;
6562
6563         /* read back registers to clear the counters */
6564         mutex_lock(&adev->grbm_idx_mutex);
6565         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6566                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6567                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6568                                 amdgpu_gfx_select_se_sh(adev, j, 0x0, k);
6569                                 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6570                         }
6571                 }
6572         }
6573         WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6574         mutex_unlock(&adev->grbm_idx_mutex);
6575
6576         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6577         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6578         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6579         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6580         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6581         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6582         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6583         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6584
6585         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6586                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6587                 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6588         }
6589
6590         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6591                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6592                 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6593         }
6594
6595         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6596                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6597                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6598         }
6599
6600         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6601                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6602                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6603         }
6604
6605         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6606         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6607         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6608         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6609 }
6610
6611 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6612                                           void *ras_error_status)
6613 {
6614         struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6615         uint32_t sec_count = 0, ded_count = 0;
6616         uint32_t i, j, k;
6617         uint32_t reg_value;
6618
6619         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6620                 return;
6621
6622         err_data->ue_count = 0;
6623         err_data->ce_count = 0;
6624
6625         mutex_lock(&adev->grbm_idx_mutex);
6626
6627         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6628                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6629                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6630                                 amdgpu_gfx_select_se_sh(adev, j, 0, k);
6631                                 reg_value =
6632                                         RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6633                                 if (reg_value)
6634                                         gfx_v9_0_ras_error_count(adev,
6635                                                 &gfx_v9_0_edc_counter_regs[i],
6636                                                 j, k, reg_value,
6637                                                 &sec_count, &ded_count);
6638                         }
6639                 }
6640         }
6641
6642         err_data->ce_count += sec_count;
6643         err_data->ue_count += ded_count;
6644
6645         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6646         mutex_unlock(&adev->grbm_idx_mutex);
6647
6648         gfx_v9_0_query_utc_edc_status(adev, err_data);
6649 }
6650
6651 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6652 {
6653         const unsigned int cp_coher_cntl =
6654                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6655                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6656                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6657                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6658                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6659
6660         /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
6661         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6662         amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
6663         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6664         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6665         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6666         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6667         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6668 }
6669
6670 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6671                                         uint32_t pipe, bool enable)
6672 {
6673         struct amdgpu_device *adev = ring->adev;
6674         uint32_t val;
6675         uint32_t wcl_cs_reg;
6676
6677         /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6678         val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
6679
6680         switch (pipe) {
6681         case 0:
6682                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
6683                 break;
6684         case 1:
6685                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
6686                 break;
6687         case 2:
6688                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
6689                 break;
6690         case 3:
6691                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
6692                 break;
6693         default:
6694                 DRM_DEBUG("invalid pipe %d\n", pipe);
6695                 return;
6696         }
6697
6698         amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6699
6700 }
6701 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6702 {
6703         struct amdgpu_device *adev = ring->adev;
6704         uint32_t val;
6705         int i;
6706
6707
6708         /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6709          * number of gfx waves. Setting 5 bit will make sure gfx only gets
6710          * around 25% of gpu resources.
6711          */
6712         val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6713         amdgpu_ring_emit_wreg(ring,
6714                               SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
6715                               val);
6716
6717         /* Restrict waves for normal/low priority compute queues as well
6718          * to get best QoS for high priority compute jobs.
6719          *
6720          * amdgpu controls only 1st ME(0-3 CS pipes).
6721          */
6722         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6723                 if (i != ring->pipe)
6724                         gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
6725
6726         }
6727 }
6728
6729 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6730         .name = "gfx_v9_0",
6731         .early_init = gfx_v9_0_early_init,
6732         .late_init = gfx_v9_0_late_init,
6733         .sw_init = gfx_v9_0_sw_init,
6734         .sw_fini = gfx_v9_0_sw_fini,
6735         .hw_init = gfx_v9_0_hw_init,
6736         .hw_fini = gfx_v9_0_hw_fini,
6737         .suspend = gfx_v9_0_suspend,
6738         .resume = gfx_v9_0_resume,
6739         .is_idle = gfx_v9_0_is_idle,
6740         .wait_for_idle = gfx_v9_0_wait_for_idle,
6741         .soft_reset = gfx_v9_0_soft_reset,
6742         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6743         .set_powergating_state = gfx_v9_0_set_powergating_state,
6744         .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6745 };
6746
6747 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6748         .type = AMDGPU_RING_TYPE_GFX,
6749         .align_mask = 0xff,
6750         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6751         .support_64bit_ptrs = true,
6752         .secure_submission_supported = true,
6753         .vmhub = AMDGPU_GFXHUB_0,
6754         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6755         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6756         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6757         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6758                 5 +  /* COND_EXEC */
6759                 7 +  /* PIPELINE_SYNC */
6760                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6761                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6762                 2 + /* VM_FLUSH */
6763                 8 +  /* FENCE for VM_FLUSH */
6764                 20 + /* GDS switch */
6765                 4 + /* double SWITCH_BUFFER,
6766                        the first COND_EXEC jump to the place just
6767                            prior to this double SWITCH_BUFFER  */
6768                 5 + /* COND_EXEC */
6769                 7 +      /*     HDP_flush */
6770                 4 +      /*     VGT_flush */
6771                 14 + /* CE_META */
6772                 31 + /* DE_META */
6773                 3 + /* CNTX_CTRL */
6774                 5 + /* HDP_INVL */
6775                 8 + 8 + /* FENCE x2 */
6776                 2 + /* SWITCH_BUFFER */
6777                 7, /* gfx_v9_0_emit_mem_sync */
6778         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6779         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6780         .emit_fence = gfx_v9_0_ring_emit_fence,
6781         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6782         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6783         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6784         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6785         .test_ring = gfx_v9_0_ring_test_ring,
6786         .insert_nop = amdgpu_ring_insert_nop,
6787         .pad_ib = amdgpu_ring_generic_pad_ib,
6788         .emit_switch_buffer = gfx_v9_ring_emit_sb,
6789         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6790         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6791         .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6792         .preempt_ib = gfx_v9_0_ring_preempt_ib,
6793         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6794         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6795         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6796         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6797         .soft_recovery = gfx_v9_0_ring_soft_recovery,
6798         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6799 };
6800
6801 static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {
6802         .type = AMDGPU_RING_TYPE_GFX,
6803         .align_mask = 0xff,
6804         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6805         .support_64bit_ptrs = true,
6806         .secure_submission_supported = true,
6807         .vmhub = AMDGPU_GFXHUB_0,
6808         .get_rptr = amdgpu_sw_ring_get_rptr_gfx,
6809         .get_wptr = amdgpu_sw_ring_get_wptr_gfx,
6810         .set_wptr = amdgpu_sw_ring_set_wptr_gfx,
6811         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6812                 5 +  /* COND_EXEC */
6813                 7 +  /* PIPELINE_SYNC */
6814                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6815                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6816                 2 + /* VM_FLUSH */
6817                 8 +  /* FENCE for VM_FLUSH */
6818                 20 + /* GDS switch */
6819                 4 + /* double SWITCH_BUFFER,
6820                      * the first COND_EXEC jump to the place just
6821                      * prior to this double SWITCH_BUFFER
6822                      */
6823                 5 + /* COND_EXEC */
6824                 7 +      /*     HDP_flush */
6825                 4 +      /*     VGT_flush */
6826                 14 + /* CE_META */
6827                 31 + /* DE_META */
6828                 3 + /* CNTX_CTRL */
6829                 5 + /* HDP_INVL */
6830                 8 + 8 + /* FENCE x2 */
6831                 2 + /* SWITCH_BUFFER */
6832                 7, /* gfx_v9_0_emit_mem_sync */
6833         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6834         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6835         .emit_fence = gfx_v9_0_ring_emit_fence,
6836         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6837         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6838         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6839         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6840         .test_ring = gfx_v9_0_ring_test_ring,
6841         .test_ib = gfx_v9_0_ring_test_ib,
6842         .insert_nop = amdgpu_sw_ring_insert_nop,
6843         .pad_ib = amdgpu_ring_generic_pad_ib,
6844         .emit_switch_buffer = gfx_v9_ring_emit_sb,
6845         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6846         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6847         .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6848         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6849         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6850         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6851         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6852         .soft_recovery = gfx_v9_0_ring_soft_recovery,
6853         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6854 };
6855
6856 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6857         .type = AMDGPU_RING_TYPE_COMPUTE,
6858         .align_mask = 0xff,
6859         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6860         .support_64bit_ptrs = true,
6861         .vmhub = AMDGPU_GFXHUB_0,
6862         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6863         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6864         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6865         .emit_frame_size =
6866                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6867                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6868                 5 + /* hdp invalidate */
6869                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6870                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6871                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6872                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6873                 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6874                 7 + /* gfx_v9_0_emit_mem_sync */
6875                 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6876                 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6877         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6878         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6879         .emit_fence = gfx_v9_0_ring_emit_fence,
6880         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6881         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6882         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6883         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6884         .test_ring = gfx_v9_0_ring_test_ring,
6885         .test_ib = gfx_v9_0_ring_test_ib,
6886         .insert_nop = amdgpu_ring_insert_nop,
6887         .pad_ib = amdgpu_ring_generic_pad_ib,
6888         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6889         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6890         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6891         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6892         .emit_wave_limit = gfx_v9_0_emit_wave_limit,
6893 };
6894
6895 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6896         .type = AMDGPU_RING_TYPE_KIQ,
6897         .align_mask = 0xff,
6898         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6899         .support_64bit_ptrs = true,
6900         .vmhub = AMDGPU_GFXHUB_0,
6901         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6902         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6903         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6904         .emit_frame_size =
6905                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6906                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6907                 5 + /* hdp invalidate */
6908                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6909                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6910                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6911                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6912                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6913         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6914         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6915         .test_ring = gfx_v9_0_ring_test_ring,
6916         .insert_nop = amdgpu_ring_insert_nop,
6917         .pad_ib = amdgpu_ring_generic_pad_ib,
6918         .emit_rreg = gfx_v9_0_ring_emit_rreg,
6919         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6920         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6921         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6922 };
6923
6924 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6925 {
6926         int i;
6927
6928         adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
6929
6930         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6931                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
6932
6933         if (adev->gfx.num_gfx_rings) {
6934                 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
6935                         adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;
6936         }
6937
6938         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6939                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
6940 }
6941
6942 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
6943         .set = gfx_v9_0_set_eop_interrupt_state,
6944         .process = gfx_v9_0_eop_irq,
6945 };
6946
6947 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
6948         .set = gfx_v9_0_set_priv_reg_fault_state,
6949         .process = gfx_v9_0_priv_reg_irq,
6950 };
6951
6952 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
6953         .set = gfx_v9_0_set_priv_inst_fault_state,
6954         .process = gfx_v9_0_priv_inst_irq,
6955 };
6956
6957 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
6958         .set = gfx_v9_0_set_cp_ecc_error_state,
6959         .process = amdgpu_gfx_cp_ecc_error_irq,
6960 };
6961
6962
6963 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
6964 {
6965         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6966         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
6967
6968         adev->gfx.priv_reg_irq.num_types = 1;
6969         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
6970
6971         adev->gfx.priv_inst_irq.num_types = 1;
6972         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
6973
6974         adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
6975         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
6976 }
6977
6978 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
6979 {
6980         switch (adev->ip_versions[GC_HWIP][0]) {
6981         case IP_VERSION(9, 0, 1):
6982         case IP_VERSION(9, 2, 1):
6983         case IP_VERSION(9, 4, 0):
6984         case IP_VERSION(9, 2, 2):
6985         case IP_VERSION(9, 1, 0):
6986         case IP_VERSION(9, 4, 1):
6987         case IP_VERSION(9, 3, 0):
6988         case IP_VERSION(9, 4, 2):
6989                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
6990                 break;
6991         default:
6992                 break;
6993         }
6994 }
6995
6996 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
6997 {
6998         /* init asci gds info */
6999         switch (adev->ip_versions[GC_HWIP][0]) {
7000         case IP_VERSION(9, 0, 1):
7001         case IP_VERSION(9, 2, 1):
7002         case IP_VERSION(9, 4, 0):
7003                 adev->gds.gds_size = 0x10000;
7004                 break;
7005         case IP_VERSION(9, 2, 2):
7006         case IP_VERSION(9, 1, 0):
7007         case IP_VERSION(9, 4, 1):
7008                 adev->gds.gds_size = 0x1000;
7009                 break;
7010         case IP_VERSION(9, 4, 2):
7011                 /* aldebaran removed all the GDS internal memory,
7012                  * only support GWS opcode in kernel, like barrier
7013                  * semaphore.etc */
7014                 adev->gds.gds_size = 0;
7015                 break;
7016         default:
7017                 adev->gds.gds_size = 0x10000;
7018                 break;
7019         }
7020
7021         switch (adev->ip_versions[GC_HWIP][0]) {
7022         case IP_VERSION(9, 0, 1):
7023         case IP_VERSION(9, 4, 0):
7024                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7025                 break;
7026         case IP_VERSION(9, 2, 1):
7027                 adev->gds.gds_compute_max_wave_id = 0x27f;
7028                 break;
7029         case IP_VERSION(9, 2, 2):
7030         case IP_VERSION(9, 1, 0):
7031                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7032                         adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7033                 else
7034                         adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7035                 break;
7036         case IP_VERSION(9, 4, 1):
7037                 adev->gds.gds_compute_max_wave_id = 0xfff;
7038                 break;
7039         case IP_VERSION(9, 4, 2):
7040                 /* deprecated for Aldebaran, no usage at all */
7041                 adev->gds.gds_compute_max_wave_id = 0;
7042                 break;
7043         default:
7044                 /* this really depends on the chip */
7045                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7046                 break;
7047         }
7048
7049         adev->gds.gws_size = 64;
7050         adev->gds.oa_size = 16;
7051 }
7052
7053 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7054                                                  u32 bitmap)
7055 {
7056         u32 data;
7057
7058         if (!bitmap)
7059                 return;
7060
7061         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7062         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7063
7064         WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7065 }
7066
7067 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7068 {
7069         u32 data, mask;
7070
7071         data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7072         data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7073
7074         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7075         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7076
7077         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7078
7079         return (~data) & mask;
7080 }
7081
7082 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7083                                  struct amdgpu_cu_info *cu_info)
7084 {
7085         int i, j, k, counter, active_cu_number = 0;
7086         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7087         unsigned disable_masks[4 * 4];
7088
7089         if (!adev || !cu_info)
7090                 return -EINVAL;
7091
7092         /*
7093          * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7094          */
7095         if (adev->gfx.config.max_shader_engines *
7096                 adev->gfx.config.max_sh_per_se > 16)
7097                 return -EINVAL;
7098
7099         amdgpu_gfx_parse_disable_cu(disable_masks,
7100                                     adev->gfx.config.max_shader_engines,
7101                                     adev->gfx.config.max_sh_per_se);
7102
7103         mutex_lock(&adev->grbm_idx_mutex);
7104         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7105                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7106                         mask = 1;
7107                         ao_bitmap = 0;
7108                         counter = 0;
7109                         amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff);
7110                         gfx_v9_0_set_user_cu_inactive_bitmap(
7111                                 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7112                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7113
7114                         /*
7115                          * The bitmap(and ao_cu_bitmap) in cu_info structure is
7116                          * 4x4 size array, and it's usually suitable for Vega
7117                          * ASICs which has 4*2 SE/SH layout.
7118                          * But for Arcturus, SE/SH layout is changed to 8*1.
7119                          * To mostly reduce the impact, we make it compatible
7120                          * with current bitmap array as below:
7121                          *    SE4,SH0 --> bitmap[0][1]
7122                          *    SE5,SH0 --> bitmap[1][1]
7123                          *    SE6,SH0 --> bitmap[2][1]
7124                          *    SE7,SH0 --> bitmap[3][1]
7125                          */
7126                         cu_info->bitmap[i % 4][j + i / 4] = bitmap;
7127
7128                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7129                                 if (bitmap & mask) {
7130                                         if (counter < adev->gfx.config.max_cu_per_sh)
7131                                                 ao_bitmap |= mask;
7132                                         counter ++;
7133                                 }
7134                                 mask <<= 1;
7135                         }
7136                         active_cu_number += counter;
7137                         if (i < 2 && j < 2)
7138                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7139                         cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7140                 }
7141         }
7142         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7143         mutex_unlock(&adev->grbm_idx_mutex);
7144
7145         cu_info->number = active_cu_number;
7146         cu_info->ao_cu_mask = ao_cu_mask;
7147         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7148
7149         return 0;
7150 }
7151
7152 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7153 {
7154         .type = AMD_IP_BLOCK_TYPE_GFX,
7155         .major = 9,
7156         .minor = 0,
7157         .rev = 0,
7158         .funcs = &gfx_v9_0_ip_funcs,
7159 };