drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "soc15.h"
  33 #include "soc15d.h"
  34 #include "amdgpu_atomfirmware.h"
  35 #include "amdgpu_pm.h"
  36
  37 #include "gc/gc_9_0_offset.h"
  38 #include "gc/gc_9_0_sh_mask.h"
  39
  40 #include "vega10_enum.h"
  41
  42 #include "soc15_common.h"
  43 #include "clearstate_gfx9.h"
  44 #include "v9_structs.h"
  45
  46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
  47
  48 #include "amdgpu_ras.h"
  49
  50 #include "gfx_v9_4.h"
  51 #include "gfx_v9_0.h"
  52 #include "gfx_v9_4_2.h"
  53
  54 #include "asic_reg/pwr/pwr_10_0_offset.h"
  55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
  56 #include "asic_reg/gc/gc_9_0_default.h"
  57
  58 #define GFX9_NUM_GFX_RINGS     1
  59 #define GFX9_MEC_HPD_SIZE 4096
  60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
  61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
  62
  63 #define mmGCEA_PROBE_MAP                        0x070c
  64 #define mmGCEA_PROBE_MAP_BASE_IDX               0
  65
  66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  68 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  72
  73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
  74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
  75 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
  76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
  77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
  78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
  79
  80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
  81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
  82 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
  83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
  84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
  85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
  86
  87 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
  88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
  89 MODULE_FIRMWARE("amdgpu/raven_me.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
  91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
  92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
  93
  94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
  95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
  96 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
  97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
  98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
  99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
 100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
 101
 102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
 103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
 104 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
 105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
 106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
 107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
 108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
 109
 110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
 112
 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
 118
 119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
 120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
 121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
 122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
 124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
 125
 126 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
 127 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
 128 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
 129
 130 #define mmTCP_CHAN_STEER_0_ARCT                                                         0x0b03
 131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX                                                        0
 132 #define mmTCP_CHAN_STEER_1_ARCT                                                         0x0b04
 133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX                                                        0
 134 #define mmTCP_CHAN_STEER_2_ARCT                                                         0x0b09
 135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX                                                        0
 136 #define mmTCP_CHAN_STEER_3_ARCT                                                         0x0b0a
 137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX                                                        0
 138 #define mmTCP_CHAN_STEER_4_ARCT                                                         0x0b0b
 139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX                                                        0
 140 #define mmTCP_CHAN_STEER_5_ARCT                                                         0x0b0c
 141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX                                                        0
 142
 143 enum ta_ras_gfx_subblock {
 144         /*CPC*/
 145         TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
 146         TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
 147         TA_RAS_BLOCK__GFX_CPC_UCODE,
 148         TA_RAS_BLOCK__GFX_DC_STATE_ME1,
 149         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
 150         TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
 151         TA_RAS_BLOCK__GFX_DC_STATE_ME2,
 152         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
 153         TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 154         TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 155         /* CPF*/
 156         TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 157         TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 158         TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
 159         TA_RAS_BLOCK__GFX_CPF_TAG,
 160         TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
 161         /* CPG*/
 162         TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 163         TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 164         TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
 165         TA_RAS_BLOCK__GFX_CPG_TAG,
 166         TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
 167         /* GDS*/
 168         TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 169         TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 170         TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
 171         TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
 172         TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
 173         TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 174         TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 175         /* SPI*/
 176         TA_RAS_BLOCK__GFX_SPI_SR_MEM,
 177         /* SQ*/
 178         TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 179         TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 180         TA_RAS_BLOCK__GFX_SQ_LDS_D,
 181         TA_RAS_BLOCK__GFX_SQ_LDS_I,
 182         TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
 183         TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
 184         /* SQC (3 ranges)*/
 185         TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 186         /* SQC range 0*/
 187         TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 188         TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
 189                 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
 190         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
 191         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
 192         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
 193         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
 194         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
 195         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 196         TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
 197                 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 198         /* SQC range 1*/
 199         TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 200         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
 201                 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 202         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
 203         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
 204         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
 205         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
 206         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
 207         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
 208         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
 209         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 210         TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
 211                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 212         /* SQC range 2*/
 213         TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 214         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
 215                 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 216         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
 217         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
 218         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
 219         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
 220         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
 221         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
 222         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
 223         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 224         TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
 225                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 226         TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
 227         /* TA*/
 228         TA_RAS_BLOCK__GFX_TA_INDEX_START,
 229         TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
 230         TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
 231         TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
 232         TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
 233         TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 234         TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 235         /* TCA*/
 236         TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 237         TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 238         TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 239         TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 240         /* TCC (5 sub-ranges)*/
 241         TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 242         /* TCC range 0*/
 243         TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 244         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
 245         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
 246         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
 247         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
 248         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
 249         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
 250         TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
 251         TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 252         TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 253         /* TCC range 1*/
 254         TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 255         TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 256         TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 257         TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
 258                 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 259         /* TCC range 2*/
 260         TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 261         TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 262         TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
 263         TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
 264         TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
 265         TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
 266         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
 267         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
 268         TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 269         TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
 270                 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 271         /* TCC range 3*/
 272         TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 273         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 274         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 275         TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
 276                 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 277         /* TCC range 4*/
 278         TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 279         TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
 280                 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 281         TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 282         TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
 283                 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 284         TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
 285         /* TCI*/
 286         TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
 287         /* TCP*/
 288         TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 289         TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 290         TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
 291         TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
 292         TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
 293         TA_RAS_BLOCK__GFX_TCP_DB_RAM,
 294         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
 295         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 296         TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 297         /* TD*/
 298         TA_RAS_BLOCK__GFX_TD_INDEX_START,
 299         TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
 300         TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
 301         TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 302         TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 303         /* EA (3 sub-ranges)*/
 304         TA_RAS_BLOCK__GFX_EA_INDEX_START,
 305         /* EA range 0*/
 306         TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
 307         TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
 308         TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
 309         TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
 310         TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
 311         TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
 312         TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
 313         TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
 314         TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 315         TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 316         /* EA range 1*/
 317         TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 318         TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 319         TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
 320         TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
 321         TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
 322         TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
 323         TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
 324         TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 325         TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 326         /* EA range 2*/
 327         TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 328         TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 329         TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
 330         TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
 331         TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 332         TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 333         TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
 334         /* UTC VM L2 bank*/
 335         TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
 336         /* UTC VM walker*/
 337         TA_RAS_BLOCK__UTC_VML2_WALKER,
 338         /* UTC ATC L2 2MB cache*/
 339         TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
 340         /* UTC ATC L2 4KB cache*/
 341         TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
 342         TA_RAS_BLOCK__GFX_MAX
 343 };
 344
 345 struct ras_gfx_subblock {
 346         unsigned char *name;
 347         int ta_subblock;
 348         int hw_supported_error_type;
 349         int sw_supported_error_type;
 350 };
 351
 352 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \
 353         [AMDGPU_RAS_BLOCK__##subblock] = {                                     \
 354                 #subblock,                                                     \
 355                 TA_RAS_BLOCK__##subblock,                                      \
 356                 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \
 357                 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \
 358         }
 359
 360 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
 361         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
 362         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
 363         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 364         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 365         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 366         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 367         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 368         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 369         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 370         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 371         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
 372         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
 373         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
 374         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
 375         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 376         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
 377         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
 378                              0),
 379         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
 380                              0),
 381         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 382         AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 383         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 384         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
 385         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
 386         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 387         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
 388         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 389                              0, 0),
 390         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 391                              0),
 392         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 393                              0, 0),
 394         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
 395                              0),
 396         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 397                              0, 0),
 398         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 399                              0),
 400         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 401                              1),
 402         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 403                              0, 0, 0),
 404         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 405                              0),
 406         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 407                              0),
 408         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 409                              0),
 410         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 411                              0),
 412         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 413                              0),
 414         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 415                              0, 0),
 416         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 417                              0),
 418         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 419                              0),
 420         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 421                              0, 0, 0),
 422         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 423                              0),
 424         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 425                              0),
 426         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 427                              0),
 428         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 429                              0),
 430         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 431                              0),
 432         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 433                              0, 0),
 434         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 435                              0),
 436         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
 437         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 438         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 439         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 440         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 441         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
 442         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 443         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
 444         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
 445                              1),
 446         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
 447                              1),
 448         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
 449                              1),
 450         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
 451                              0),
 452         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
 453                              0),
 454         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 455         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 456         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
 457         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
 458         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
 459         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
 460         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 461         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
 462         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
 463         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 464         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
 465         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
 466                              0),
 467         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 468         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
 469                              0),
 470         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
 471                              0, 0),
 472         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
 473                              0),
 474         AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 475         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
 476         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
 477         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 478         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 479         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 480         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
 481         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
 482         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
 483         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
 484         AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 485         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
 486         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 487         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 488         AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 489         AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 490         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 491         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 492         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 493         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 494         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 495         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 496         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 497         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 498         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 499         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 500         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 501         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 502         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 503         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 504         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
 505         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
 506         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
 507         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
 508 };
 509
 510 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
 511 {
 512         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 513         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
 514         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 515         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 516         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 517         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 518         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 519         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 520         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 521         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
 522         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
 523         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 524         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 525         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 526         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 527         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 528         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
 529         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 530         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 531         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 532 };
 533
 534 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 535 {
 536         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 537         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 538         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 539         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 540         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 541         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 542         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
 543         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 544         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
 545         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 546         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 547         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 548         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 549         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 550         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 551         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
 552         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
 553         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 554 };
 555
 556 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
 557 {
 558         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
 559         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 560         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 561         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
 562         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
 563         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
 564         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
 565         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
 566         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
 567         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
 568         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
 569 };
 570
 571 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 572 {
 573         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 574         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 575         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 576         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 577         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 578         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 579         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 580         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 581         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 582         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 583         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 584         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 585         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 586         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 587         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 588         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 589         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 590         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 591         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 592         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
 593         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
 594         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 595         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 596         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 597 };
 598
 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 600 {
 601         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 602         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
 603         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
 604         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
 605         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
 606         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 607         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 608 };
 609
 610 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
 611 {
 612         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
 613         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 614         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 615         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
 616         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
 617         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
 618         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
 619         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
 620         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 621         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 622         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
 623         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
 624         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
 625         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 626         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 627         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 628         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
 629         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 630         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 631 };
 632
 633 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
 634 {
 635         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 636         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 637         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 638         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
 639         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
 640         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 641         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 642         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 643         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 644         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 645         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 646         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
 647 };
 648
 649 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 650 {
 651         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
 652         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
 653         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 654 };
 655
 656 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
 657 {
 658         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 659         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 660         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 661         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 662         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 663         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 664         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 665         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 666         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
 667         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 668         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 669         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 670         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 671         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 672         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 673         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 674 };
 675
 676 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 677 {
 678         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
 679         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 680         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 681         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
 682         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
 683         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 684         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
 685         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 686         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
 687         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 688         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 689         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 690         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 691 };
 692
 693 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
 694 {
 695         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 696         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
 697         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
 698         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
 699         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
 700         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
 701         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
 702         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
 703         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
 704         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
 705         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
 706 };
 707
 708 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
 709         {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
 710         {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
 711 };
 712
 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
 714 {
 715         mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 716         mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 717         mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 718         mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 719         mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 720         mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 721         mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 722         mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 723 };
 724
 725 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 726 {
 727         mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 728         mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 729         mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 730         mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 731         mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 732         mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 733         mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 734         mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 735 };
 736
 737 static void gfx_v9_0_rlcg_w(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag)
 738 {
 739         static void *scratch_reg0;
 740         static void *scratch_reg1;
 741         static void *scratch_reg2;
 742         static void *scratch_reg3;
 743         static void *spare_int;
 744         static uint32_t grbm_cntl;
 745         static uint32_t grbm_idx;
 746
 747         scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4;
 748         scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4;
 749         scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4;
 750         scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4;
 751         spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4;
 752
 753         grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;
 754         grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;
 755
 756         if (amdgpu_sriov_runtime(adev)) {
 757                 pr_err("shouldn't call rlcg write register during runtime\n");
 758                 return;
 759         }
 760
 761         if (offset == grbm_cntl || offset == grbm_idx) {
 762                 if (offset  == grbm_cntl)
 763                         writel(v, scratch_reg2);
 764                 else if (offset == grbm_idx)
 765                         writel(v, scratch_reg3);
 766
 767                 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
 768         } else {
 769                 uint32_t i = 0;
 770                 uint32_t retries = 50000;
 771
 772                 writel(v, scratch_reg0);
 773                 writel(offset | 0x80000000, scratch_reg1);
 774                 writel(1, spare_int);
 775                 for (i = 0; i < retries; i++) {
 776                         u32 tmp;
 777
 778                         tmp = readl(scratch_reg1);
 779                         if (!(tmp & 0x80000000))
 780                                 break;
 781
 782                         udelay(10);
 783                 }
 784                 if (i >= retries)
 785                         pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
 786         }
 787
 788 }
 789
 790 static void gfx_v9_0_sriov_wreg(struct amdgpu_device *adev, u32 offset,
 791                                u32 v, u32 acc_flags, u32 hwip)
 792 {
 793         if ((acc_flags & AMDGPU_REGS_RLC) &&
 794             amdgpu_sriov_fullaccess(adev)) {
 795                 gfx_v9_0_rlcg_w(adev, offset, v, acc_flags);
 796
 797                 return;
 798         }
 799
 800         if (acc_flags & AMDGPU_REGS_NO_KIQ)
 801                 WREG32_NO_KIQ(offset, v);
 802         else
 803                 WREG32(offset, v);
 804 }
 805
 806 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 807 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 808 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 809 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
 810
 811 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 812 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 813 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 814 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 815 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 816                                 struct amdgpu_cu_info *cu_info);
 817 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 818 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
 819 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
 820 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
 821                                           void *ras_error_status);
 822 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
 823                                      void *inject_if);
 824 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
 825
 826 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 827                                 uint64_t queue_mask)
 828 {
 829         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 830         amdgpu_ring_write(kiq_ring,
 831                 PACKET3_SET_RESOURCES_VMID_MASK(0) |
 832                 /* vmid_mask:0* queue_type:0 (KIQ) */
 833                 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
 834         amdgpu_ring_write(kiq_ring,
 835                         lower_32_bits(queue_mask));     /* queue mask lo */
 836         amdgpu_ring_write(kiq_ring,
 837                         upper_32_bits(queue_mask));     /* queue mask hi */
 838         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 839         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 840         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 841         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 842 }
 843
 844 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 845                                  struct amdgpu_ring *ring)
 846 {
 847         struct amdgpu_device *adev = kiq_ring->adev;
 848         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 849         uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
 850         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 851
 852         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 853         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 854         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 855                          PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 856                          PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 857                          PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 858                          PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 859                          PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
 860                          /*queue_type: normal compute queue */
 861                          PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
 862                          /* alloc format: all_on_one_pipe */
 863                          PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
 864                          PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 865                          /* num_queues: must be 1 */
 866                          PACKET3_MAP_QUEUES_NUM_QUEUES(1));
 867         amdgpu_ring_write(kiq_ring,
 868                         PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 869         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 870         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 871         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 872         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 873 }
 874
 875 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 876                                    struct amdgpu_ring *ring,
 877                                    enum amdgpu_unmap_queues_action action,
 878                                    u64 gpu_addr, u64 seq)
 879 {
 880         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 881
 882         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 883         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 884                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 885                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 886                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 887                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 888         amdgpu_ring_write(kiq_ring,
 889                         PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 890
 891         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 892                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 893                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 894                 amdgpu_ring_write(kiq_ring, seq);
 895         } else {
 896                 amdgpu_ring_write(kiq_ring, 0);
 897                 amdgpu_ring_write(kiq_ring, 0);
 898                 amdgpu_ring_write(kiq_ring, 0);
 899         }
 900 }
 901
 902 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 903                                    struct amdgpu_ring *ring,
 904                                    u64 addr,
 905                                    u64 seq)
 906 {
 907         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 908
 909         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 910         amdgpu_ring_write(kiq_ring,
 911                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 912                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 913                           PACKET3_QUERY_STATUS_COMMAND(2));
 914         /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 915         amdgpu_ring_write(kiq_ring,
 916                         PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 917                         PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 918         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 919         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 920         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 921         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 922 }
 923
 924 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 925                                 uint16_t pasid, uint32_t flush_type,
 926                                 bool all_hub)
 927 {
 928         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
 929         amdgpu_ring_write(kiq_ring,
 930                         PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
 931                         PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
 932                         PACKET3_INVALIDATE_TLBS_PASID(pasid) |
 933                         PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
 934 }
 935
 936 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
 937         .kiq_set_resources = gfx_v9_0_kiq_set_resources,
 938         .kiq_map_queues = gfx_v9_0_kiq_map_queues,
 939         .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
 940         .kiq_query_status = gfx_v9_0_kiq_query_status,
 941         .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
 942         .set_resources_size = 8,
 943         .map_queues_size = 7,
 944         .unmap_queues_size = 6,
 945         .query_status_size = 7,
 946         .invalidate_tlbs_size = 2,
 947 };
 948
 949 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 950 {
 951         adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
 952 }
 953
 954 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 955 {
 956         switch (adev->ip_versions[GC_HWIP][0]) {
 957         case IP_VERSION(9, 0, 1):
 958                 soc15_program_register_sequence(adev,
 959                                                 golden_settings_gc_9_0,
 960                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 961                 soc15_program_register_sequence(adev,
 962                                                 golden_settings_gc_9_0_vg10,
 963                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
 964                 break;
 965         case IP_VERSION(9, 2, 1):
 966                 soc15_program_register_sequence(adev,
 967                                                 golden_settings_gc_9_2_1,
 968                                                 ARRAY_SIZE(golden_settings_gc_9_2_1));
 969                 soc15_program_register_sequence(adev,
 970                                                 golden_settings_gc_9_2_1_vg12,
 971                                                 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
 972                 break;
 973         case IP_VERSION(9, 4, 0):
 974                 soc15_program_register_sequence(adev,
 975                                                 golden_settings_gc_9_0,
 976                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 977                 soc15_program_register_sequence(adev,
 978                                                 golden_settings_gc_9_0_vg20,
 979                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
 980                 break;
 981         case IP_VERSION(9, 4, 1):
 982                 soc15_program_register_sequence(adev,
 983                                                 golden_settings_gc_9_4_1_arct,
 984                                                 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
 985                 break;
 986         case IP_VERSION(9, 2, 2):
 987         case IP_VERSION(9, 1, 0):
 988                 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
 989                                                 ARRAY_SIZE(golden_settings_gc_9_1));
 990                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
 991                         soc15_program_register_sequence(adev,
 992                                                         golden_settings_gc_9_1_rv2,
 993                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv2));
 994                 else
 995                         soc15_program_register_sequence(adev,
 996                                                         golden_settings_gc_9_1_rv1,
 997                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv1));
 998                 break;
 999          case IP_VERSION(9, 3, 0):
1000                 soc15_program_register_sequence(adev,
1001                                                 golden_settings_gc_9_1_rn,
1002                                                 ARRAY_SIZE(golden_settings_gc_9_1_rn));
1003                 return; /* for renoir, don't need common goldensetting */
1004         case IP_VERSION(9, 4, 2):
1005                 gfx_v9_4_2_init_golden_registers(adev,
1006                                                  adev->smuio.funcs->get_die_id(adev));
1007                 break;
1008         default:
1009                 break;
1010         }
1011
1012         if ((adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) &&
1013             (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 2)))
1014                 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
1015                                                 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
1016 }
1017
1018 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
1019 {
1020         adev->gfx.scratch.num_reg = 8;
1021         adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1022         adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
1023 }
1024
1025 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1026                                        bool wc, uint32_t reg, uint32_t val)
1027 {
1028         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1029         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1030                                 WRITE_DATA_DST_SEL(0) |
1031                                 (wc ? WR_CONFIRM : 0));
1032         amdgpu_ring_write(ring, reg);
1033         amdgpu_ring_write(ring, 0);
1034         amdgpu_ring_write(ring, val);
1035 }
1036
1037 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1038                                   int mem_space, int opt, uint32_t addr0,
1039                                   uint32_t addr1, uint32_t ref, uint32_t mask,
1040                                   uint32_t inv)
1041 {
1042         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1043         amdgpu_ring_write(ring,
1044                                  /* memory (1) or register (0) */
1045                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1046                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
1047                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
1048                                  WAIT_REG_MEM_ENGINE(eng_sel)));
1049
1050         if (mem_space)
1051                 BUG_ON(addr0 & 0x3); /* Dword align */
1052         amdgpu_ring_write(ring, addr0);
1053         amdgpu_ring_write(ring, addr1);
1054         amdgpu_ring_write(ring, ref);
1055         amdgpu_ring_write(ring, mask);
1056         amdgpu_ring_write(ring, inv); /* poll interval */
1057 }
1058
1059 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1060 {
1061         struct amdgpu_device *adev = ring->adev;
1062         uint32_t scratch;
1063         uint32_t tmp = 0;
1064         unsigned i;
1065         int r;
1066
1067         r = amdgpu_gfx_scratch_get(adev, &scratch);
1068         if (r)
1069                 return r;
1070
1071         WREG32(scratch, 0xCAFEDEAD);
1072         r = amdgpu_ring_alloc(ring, 3);
1073         if (r)
1074                 goto error_free_scratch;
1075
1076         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1077         amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
1078         amdgpu_ring_write(ring, 0xDEADBEEF);
1079         amdgpu_ring_commit(ring);
1080
1081         for (i = 0; i < adev->usec_timeout; i++) {
1082                 tmp = RREG32(scratch);
1083                 if (tmp == 0xDEADBEEF)
1084                         break;
1085                 udelay(1);
1086         }
1087
1088         if (i >= adev->usec_timeout)
1089                 r = -ETIMEDOUT;
1090
1091 error_free_scratch:
1092         amdgpu_gfx_scratch_free(adev, scratch);
1093         return r;
1094 }
1095
1096 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1097 {
1098         struct amdgpu_device *adev = ring->adev;
1099         struct amdgpu_ib ib;
1100         struct dma_fence *f = NULL;
1101
1102         unsigned index;
1103         uint64_t gpu_addr;
1104         uint32_t tmp;
1105         long r;
1106
1107         r = amdgpu_device_wb_get(adev, &index);
1108         if (r)
1109                 return r;
1110
1111         gpu_addr = adev->wb.gpu_addr + (index * 4);
1112         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1113         memset(&ib, 0, sizeof(ib));
1114         r = amdgpu_ib_get(adev, NULL, 16,
1115                                         AMDGPU_IB_POOL_DIRECT, &ib);
1116         if (r)
1117                 goto err1;
1118
1119         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1120         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1121         ib.ptr[2] = lower_32_bits(gpu_addr);
1122         ib.ptr[3] = upper_32_bits(gpu_addr);
1123         ib.ptr[4] = 0xDEADBEEF;
1124         ib.length_dw = 5;
1125
1126         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1127         if (r)
1128                 goto err2;
1129
1130         r = dma_fence_wait_timeout(f, false, timeout);
1131         if (r == 0) {
1132                 r = -ETIMEDOUT;
1133                 goto err2;
1134         } else if (r < 0) {
1135                 goto err2;
1136         }
1137
1138         tmp = adev->wb.wb[index];
1139         if (tmp == 0xDEADBEEF)
1140                 r = 0;
1141         else
1142                 r = -EINVAL;
1143
1144 err2:
1145         amdgpu_ib_free(adev, &ib, NULL);
1146         dma_fence_put(f);
1147 err1:
1148         amdgpu_device_wb_free(adev, index);
1149         return r;
1150 }
1151
1152
1153 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1154 {
1155         release_firmware(adev->gfx.pfp_fw);
1156         adev->gfx.pfp_fw = NULL;
1157         release_firmware(adev->gfx.me_fw);
1158         adev->gfx.me_fw = NULL;
1159         release_firmware(adev->gfx.ce_fw);
1160         adev->gfx.ce_fw = NULL;
1161         release_firmware(adev->gfx.rlc_fw);
1162         adev->gfx.rlc_fw = NULL;
1163         release_firmware(adev->gfx.mec_fw);
1164         adev->gfx.mec_fw = NULL;
1165         release_firmware(adev->gfx.mec2_fw);
1166         adev->gfx.mec2_fw = NULL;
1167
1168         kfree(adev->gfx.rlc.register_list_format);
1169 }
1170
1171 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
1172 {
1173         const struct rlc_firmware_header_v2_1 *rlc_hdr;
1174
1175         rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1176         adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
1177         adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
1178         adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
1179         adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
1180         adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
1181         adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
1182         adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
1183         adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
1184         adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
1185         adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
1186         adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
1187         adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
1188         adev->gfx.rlc.reg_list_format_direct_reg_list_length =
1189                         le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
1190 }
1191
1192 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1193 {
1194         adev->gfx.me_fw_write_wait = false;
1195         adev->gfx.mec_fw_write_wait = false;
1196
1197         if ((adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) &&
1198             ((adev->gfx.mec_fw_version < 0x000001a5) ||
1199             (adev->gfx.mec_feature_version < 46) ||
1200             (adev->gfx.pfp_fw_version < 0x000000b7) ||
1201             (adev->gfx.pfp_feature_version < 46)))
1202                 DRM_WARN_ONCE("CP firmware version too old, please update!");
1203
1204         switch (adev->ip_versions[GC_HWIP][0]) {
1205         case IP_VERSION(9, 0, 1):
1206                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1207                     (adev->gfx.me_feature_version >= 42) &&
1208                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1209                     (adev->gfx.pfp_feature_version >= 42))
1210                         adev->gfx.me_fw_write_wait = true;
1211
1212                 if ((adev->gfx.mec_fw_version >=  0x00000193) &&
1213                     (adev->gfx.mec_feature_version >= 42))
1214                         adev->gfx.mec_fw_write_wait = true;
1215                 break;
1216         case IP_VERSION(9, 2, 1):
1217                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1218                     (adev->gfx.me_feature_version >= 44) &&
1219                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1220                     (adev->gfx.pfp_feature_version >= 44))
1221                         adev->gfx.me_fw_write_wait = true;
1222
1223                 if ((adev->gfx.mec_fw_version >=  0x00000196) &&
1224                     (adev->gfx.mec_feature_version >= 44))
1225                         adev->gfx.mec_fw_write_wait = true;
1226                 break;
1227         case IP_VERSION(9, 4, 0):
1228                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1229                     (adev->gfx.me_feature_version >= 44) &&
1230                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1231                     (adev->gfx.pfp_feature_version >= 44))
1232                         adev->gfx.me_fw_write_wait = true;
1233
1234                 if ((adev->gfx.mec_fw_version >=  0x00000197) &&
1235                     (adev->gfx.mec_feature_version >= 44))
1236                         adev->gfx.mec_fw_write_wait = true;
1237                 break;
1238         case IP_VERSION(9, 1, 0):
1239         case IP_VERSION(9, 2, 2):
1240                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1241                     (adev->gfx.me_feature_version >= 42) &&
1242                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1243                     (adev->gfx.pfp_feature_version >= 42))
1244                         adev->gfx.me_fw_write_wait = true;
1245
1246                 if ((adev->gfx.mec_fw_version >=  0x00000192) &&
1247                     (adev->gfx.mec_feature_version >= 42))
1248                         adev->gfx.mec_fw_write_wait = true;
1249                 break;
1250         default:
1251                 adev->gfx.me_fw_write_wait = true;
1252                 adev->gfx.mec_fw_write_wait = true;
1253                 break;
1254         }
1255 }
1256
1257 struct amdgpu_gfxoff_quirk {
1258         u16 chip_vendor;
1259         u16 chip_device;
1260         u16 subsys_vendor;
1261         u16 subsys_device;
1262         u8 revision;
1263 };
1264
1265 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1266         /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1267         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1268         /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1269         { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1270         /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1271         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1272         { 0, 0, 0, 0, 0 },
1273 };
1274
1275 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1276 {
1277         const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1278
1279         while (p && p->chip_device != 0) {
1280                 if (pdev->vendor == p->chip_vendor &&
1281                     pdev->device == p->chip_device &&
1282                     pdev->subsystem_vendor == p->subsys_vendor &&
1283                     pdev->subsystem_device == p->subsys_device &&
1284                     pdev->revision == p->revision) {
1285                         return true;
1286                 }
1287                 ++p;
1288         }
1289         return false;
1290 }
1291
1292 static bool is_raven_kicker(struct amdgpu_device *adev)
1293 {
1294         if (adev->pm.fw_version >= 0x41e2b)
1295                 return true;
1296         else
1297                 return false;
1298 }
1299
1300 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1301 {
1302         if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 3, 0)) &&
1303             (adev->gfx.me_fw_version >= 0x000000a5) &&
1304             (adev->gfx.me_feature_version >= 52))
1305                 return true;
1306         else
1307                 return false;
1308 }
1309
1310 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1311 {
1312         if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1313                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1314
1315         switch (adev->ip_versions[GC_HWIP][0]) {
1316         case IP_VERSION(9, 0, 1):
1317         case IP_VERSION(9, 2, 1):
1318         case IP_VERSION(9, 4, 0):
1319                 break;
1320         case IP_VERSION(9, 2, 2):
1321         case IP_VERSION(9, 1, 0):
1322                 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1323                       (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1324                     ((!is_raven_kicker(adev) &&
1325                       adev->gfx.rlc_fw_version < 531) ||
1326                      (adev->gfx.rlc_feature_version < 1) ||
1327                      !adev->gfx.rlc.is_rlc_v2_1))
1328                         adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1329
1330                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1331                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1332                                 AMD_PG_SUPPORT_CP |
1333                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1334                 break;
1335         case IP_VERSION(9, 3, 0):
1336                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1337                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1338                                 AMD_PG_SUPPORT_CP |
1339                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1340                 break;
1341         default:
1342                 break;
1343         }
1344 }
1345
1346 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1347                                           const char *chip_name)
1348 {
1349         char fw_name[30];
1350         int err;
1351         struct amdgpu_firmware_info *info = NULL;
1352         const struct common_firmware_header *header = NULL;
1353         const struct gfx_firmware_header_v1_0 *cp_hdr;
1354
1355         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1356         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1357         if (err)
1358                 goto out;
1359         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1360         if (err)
1361                 goto out;
1362         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1363         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1364         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1365
1366         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1367         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1368         if (err)
1369                 goto out;
1370         err = amdgpu_ucode_validate(adev->gfx.me_fw);
1371         if (err)
1372                 goto out;
1373         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1374         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1375         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1376
1377         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1378         err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1379         if (err)
1380                 goto out;
1381         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1382         if (err)
1383                 goto out;
1384         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1385         adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1386         adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1387
1388         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1389                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1390                 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1391                 info->fw = adev->gfx.pfp_fw;
1392                 header = (const struct common_firmware_header *)info->fw->data;
1393                 adev->firmware.fw_size +=
1394                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1395
1396                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1397                 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1398                 info->fw = adev->gfx.me_fw;
1399                 header = (const struct common_firmware_header *)info->fw->data;
1400                 adev->firmware.fw_size +=
1401                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1402
1403                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1404                 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1405                 info->fw = adev->gfx.ce_fw;
1406                 header = (const struct common_firmware_header *)info->fw->data;
1407                 adev->firmware.fw_size +=
1408                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1409         }
1410
1411 out:
1412         if (err) {
1413                 dev_err(adev->dev,
1414                         "gfx9: Failed to load firmware \"%s\"\n",
1415                         fw_name);
1416                 release_firmware(adev->gfx.pfp_fw);
1417                 adev->gfx.pfp_fw = NULL;
1418                 release_firmware(adev->gfx.me_fw);
1419                 adev->gfx.me_fw = NULL;
1420                 release_firmware(adev->gfx.ce_fw);
1421                 adev->gfx.ce_fw = NULL;
1422         }
1423         return err;
1424 }
1425
1426 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1427                                           const char *chip_name)
1428 {
1429         char fw_name[30];
1430         int err;
1431         struct amdgpu_firmware_info *info = NULL;
1432         const struct common_firmware_header *header = NULL;
1433         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1434         unsigned int *tmp = NULL;
1435         unsigned int i = 0;
1436         uint16_t version_major;
1437         uint16_t version_minor;
1438         uint32_t smu_version;
1439
1440         /*
1441          * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1442          * instead of picasso_rlc.bin.
1443          * Judgment method:
1444          * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1445          *          or revision >= 0xD8 && revision <= 0xDF
1446          * otherwise is PCO FP5
1447          */
1448         if (!strcmp(chip_name, "picasso") &&
1449                 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1450                 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1451                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1452         else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1453                 (smu_version >= 0x41e2b))
1454                 /**
1455                 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1456                 */
1457                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1458         else
1459                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1460         err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1461         if (err)
1462                 goto out;
1463         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1464         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1465
1466         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1467         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1468         if (version_major == 2 && version_minor == 1)
1469                 adev->gfx.rlc.is_rlc_v2_1 = true;
1470
1471         adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1472         adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1473         adev->gfx.rlc.save_and_restore_offset =
1474                         le32_to_cpu(rlc_hdr->save_and_restore_offset);
1475         adev->gfx.rlc.clear_state_descriptor_offset =
1476                         le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1477         adev->gfx.rlc.avail_scratch_ram_locations =
1478                         le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1479         adev->gfx.rlc.reg_restore_list_size =
1480                         le32_to_cpu(rlc_hdr->reg_restore_list_size);
1481         adev->gfx.rlc.reg_list_format_start =
1482                         le32_to_cpu(rlc_hdr->reg_list_format_start);
1483         adev->gfx.rlc.reg_list_format_separate_start =
1484                         le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1485         adev->gfx.rlc.starting_offsets_start =
1486                         le32_to_cpu(rlc_hdr->starting_offsets_start);
1487         adev->gfx.rlc.reg_list_format_size_bytes =
1488                         le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1489         adev->gfx.rlc.reg_list_size_bytes =
1490                         le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1491         adev->gfx.rlc.register_list_format =
1492                         kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1493                                 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1494         if (!adev->gfx.rlc.register_list_format) {
1495                 err = -ENOMEM;
1496                 goto out;
1497         }
1498
1499         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1500                         le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1501         for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1502                 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1503
1504         adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1505
1506         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1507                         le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1508         for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1509                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1510
1511         if (adev->gfx.rlc.is_rlc_v2_1)
1512                 gfx_v9_0_init_rlc_ext_microcode(adev);
1513
1514         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1515                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1516                 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1517                 info->fw = adev->gfx.rlc_fw;
1518                 header = (const struct common_firmware_header *)info->fw->data;
1519                 adev->firmware.fw_size +=
1520                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1521
1522                 if (adev->gfx.rlc.is_rlc_v2_1 &&
1523                     adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
1524                     adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
1525                     adev->gfx.rlc.save_restore_list_srm_size_bytes) {
1526                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
1527                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
1528                         info->fw = adev->gfx.rlc_fw;
1529                         adev->firmware.fw_size +=
1530                                 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
1531
1532                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
1533                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
1534                         info->fw = adev->gfx.rlc_fw;
1535                         adev->firmware.fw_size +=
1536                                 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
1537
1538                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
1539                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
1540                         info->fw = adev->gfx.rlc_fw;
1541                         adev->firmware.fw_size +=
1542                                 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
1543                 }
1544         }
1545
1546 out:
1547         if (err) {
1548                 dev_err(adev->dev,
1549                         "gfx9: Failed to load firmware \"%s\"\n",
1550                         fw_name);
1551                 release_firmware(adev->gfx.rlc_fw);
1552                 adev->gfx.rlc_fw = NULL;
1553         }
1554         return err;
1555 }
1556
1557 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1558 {
1559         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
1560             adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) ||
1561             adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 3, 0))
1562                 return false;
1563
1564         return true;
1565 }
1566
1567 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1568                                           const char *chip_name)
1569 {
1570         char fw_name[30];
1571         int err;
1572         struct amdgpu_firmware_info *info = NULL;
1573         const struct common_firmware_header *header = NULL;
1574         const struct gfx_firmware_header_v1_0 *cp_hdr;
1575
1576         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1577         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1578         if (err)
1579                 goto out;
1580         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1581         if (err)
1582                 goto out;
1583         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1584         adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1585         adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1586
1587
1588         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1589                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1590                 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1591                 if (!err) {
1592                         err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1593                         if (err)
1594                                 goto out;
1595                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1596                         adev->gfx.mec2_fw->data;
1597                         adev->gfx.mec2_fw_version =
1598                         le32_to_cpu(cp_hdr->header.ucode_version);
1599                         adev->gfx.mec2_feature_version =
1600                         le32_to_cpu(cp_hdr->ucode_feature_version);
1601                 } else {
1602                         err = 0;
1603                         adev->gfx.mec2_fw = NULL;
1604                 }
1605         } else {
1606                 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1607                 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1608         }
1609
1610         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1611                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1612                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1613                 info->fw = adev->gfx.mec_fw;
1614                 header = (const struct common_firmware_header *)info->fw->data;
1615                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1616                 adev->firmware.fw_size +=
1617                         ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1618
1619                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
1620                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
1621                 info->fw = adev->gfx.mec_fw;
1622                 adev->firmware.fw_size +=
1623                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1624
1625                 if (adev->gfx.mec2_fw) {
1626                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1627                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1628                         info->fw = adev->gfx.mec2_fw;
1629                         header = (const struct common_firmware_header *)info->fw->data;
1630                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1631                         adev->firmware.fw_size +=
1632                                 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1633
1634                         /* TODO: Determine if MEC2 JT FW loading can be removed
1635                                  for all GFX V9 asic and above */
1636                         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1637                                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
1638                                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
1639                                 info->fw = adev->gfx.mec2_fw;
1640                                 adev->firmware.fw_size +=
1641                                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
1642                                         PAGE_SIZE);
1643                         }
1644                 }
1645         }
1646
1647 out:
1648         gfx_v9_0_check_if_need_gfxoff(adev);
1649         gfx_v9_0_check_fw_write_wait(adev);
1650         if (err) {
1651                 dev_err(adev->dev,
1652                         "gfx9: Failed to load firmware \"%s\"\n",
1653                         fw_name);
1654                 release_firmware(adev->gfx.mec_fw);
1655                 adev->gfx.mec_fw = NULL;
1656                 release_firmware(adev->gfx.mec2_fw);
1657                 adev->gfx.mec2_fw = NULL;
1658         }
1659         return err;
1660 }
1661
1662 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1663 {
1664         const char *chip_name;
1665         int r;
1666
1667         DRM_DEBUG("\n");
1668
1669         switch (adev->ip_versions[GC_HWIP][0]) {
1670         case IP_VERSION(9, 0, 1):
1671                 chip_name = "vega10";
1672                 break;
1673         case IP_VERSION(9, 2, 1):
1674                 chip_name = "vega12";
1675                 break;
1676         case IP_VERSION(9, 4, 0):
1677                 chip_name = "vega20";
1678                 break;
1679         case IP_VERSION(9, 2, 2):
1680         case IP_VERSION(9, 1, 0):
1681                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1682                         chip_name = "raven2";
1683                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1684                         chip_name = "picasso";
1685                 else
1686                         chip_name = "raven";
1687                 break;
1688         case IP_VERSION(9, 4, 1):
1689                 chip_name = "arcturus";
1690                 break;
1691         case IP_VERSION(9, 3, 0):
1692                 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1693                         chip_name = "renoir";
1694                 else
1695                         chip_name = "green_sardine";
1696                 break;
1697         case IP_VERSION(9, 4, 2):
1698                 chip_name = "aldebaran";
1699                 break;
1700         default:
1701                 BUG();
1702         }
1703
1704         /* No CPG in Arcturus */
1705         if (adev->gfx.num_gfx_rings) {
1706                 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name);
1707                 if (r)
1708                         return r;
1709         }
1710
1711         r = gfx_v9_0_init_rlc_microcode(adev, chip_name);
1712         if (r)
1713                 return r;
1714
1715         r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name);
1716         if (r)
1717                 return r;
1718
1719         return r;
1720 }
1721
1722 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1723 {
1724         u32 count = 0;
1725         const struct cs_section_def *sect = NULL;
1726         const struct cs_extent_def *ext = NULL;
1727
1728         /* begin clear state */
1729         count += 2;
1730         /* context control state */
1731         count += 3;
1732
1733         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1734                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1735                         if (sect->id == SECT_CONTEXT)
1736                                 count += 2 + ext->reg_count;
1737                         else
1738                                 return 0;
1739                 }
1740         }
1741
1742         /* end clear state */
1743         count += 2;
1744         /* clear state */
1745         count += 2;
1746
1747         return count;
1748 }
1749
1750 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1751                                     volatile u32 *buffer)
1752 {
1753         u32 count = 0, i;
1754         const struct cs_section_def *sect = NULL;
1755         const struct cs_extent_def *ext = NULL;
1756
1757         if (adev->gfx.rlc.cs_data == NULL)
1758                 return;
1759         if (buffer == NULL)
1760                 return;
1761
1762         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1763         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1764
1765         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1766         buffer[count++] = cpu_to_le32(0x80000000);
1767         buffer[count++] = cpu_to_le32(0x80000000);
1768
1769         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1770                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1771                         if (sect->id == SECT_CONTEXT) {
1772                                 buffer[count++] =
1773                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1774                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1775                                                 PACKET3_SET_CONTEXT_REG_START);
1776                                 for (i = 0; i < ext->reg_count; i++)
1777                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1778                         } else {
1779                                 return;
1780                         }
1781                 }
1782         }
1783
1784         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1785         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1786
1787         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1788         buffer[count++] = cpu_to_le32(0);
1789 }
1790
1791 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1792 {
1793         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1794         uint32_t pg_always_on_cu_num = 2;
1795         uint32_t always_on_cu_num;
1796         uint32_t i, j, k;
1797         uint32_t mask, cu_bitmap, counter;
1798
1799         if (adev->flags & AMD_IS_APU)
1800                 always_on_cu_num = 4;
1801         else if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 2, 1))
1802                 always_on_cu_num = 8;
1803         else
1804                 always_on_cu_num = 12;
1805
1806         mutex_lock(&adev->grbm_idx_mutex);
1807         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1808                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1809                         mask = 1;
1810                         cu_bitmap = 0;
1811                         counter = 0;
1812                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1813
1814                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1815                                 if (cu_info->bitmap[i][j] & mask) {
1816                                         if (counter == pg_always_on_cu_num)
1817                                                 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1818                                         if (counter < always_on_cu_num)
1819                                                 cu_bitmap |= mask;
1820                                         else
1821                                                 break;
1822                                         counter++;
1823                                 }
1824                                 mask <<= 1;
1825                         }
1826
1827                         WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1828                         cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1829                 }
1830         }
1831         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1832         mutex_unlock(&adev->grbm_idx_mutex);
1833 }
1834
1835 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1836 {
1837         uint32_t data;
1838
1839         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1840         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1841         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1842         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1843         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1844
1845         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1846         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1847
1848         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1849         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1850
1851         mutex_lock(&adev->grbm_idx_mutex);
1852         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1853         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1854         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1855
1856         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1857         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1858         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1859         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1860         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1861
1862         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1863         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1864         data &= 0x0000FFFF;
1865         data |= 0x00C00000;
1866         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1867
1868         /*
1869          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1870          * programmed in gfx_v9_0_init_always_on_cu_mask()
1871          */
1872
1873         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1874          * but used for RLC_LB_CNTL configuration */
1875         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1876         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1877         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1878         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1879         mutex_unlock(&adev->grbm_idx_mutex);
1880
1881         gfx_v9_0_init_always_on_cu_mask(adev);
1882 }
1883
1884 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1885 {
1886         uint32_t data;
1887
1888         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1889         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1890         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1891         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1892         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1893
1894         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1895         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1896
1897         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1898         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1899
1900         mutex_lock(&adev->grbm_idx_mutex);
1901         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1902         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1903         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1904
1905         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1906         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1907         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1908         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1909         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1910
1911         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1912         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1913         data &= 0x0000FFFF;
1914         data |= 0x00C00000;
1915         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1916
1917         /*
1918          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1919          * programmed in gfx_v9_0_init_always_on_cu_mask()
1920          */
1921
1922         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1923          * but used for RLC_LB_CNTL configuration */
1924         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1925         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1926         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1927         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1928         mutex_unlock(&adev->grbm_idx_mutex);
1929
1930         gfx_v9_0_init_always_on_cu_mask(adev);
1931 }
1932
1933 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1934 {
1935         WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1936 }
1937
1938 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1939 {
1940         if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1941                 return 5;
1942         else
1943                 return 4;
1944 }
1945
1946 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1947 {
1948         const struct cs_section_def *cs_data;
1949         int r;
1950
1951         adev->gfx.rlc.cs_data = gfx9_cs_data;
1952
1953         cs_data = adev->gfx.rlc.cs_data;
1954
1955         if (cs_data) {
1956                 /* init clear state block */
1957                 r = amdgpu_gfx_rlc_init_csb(adev);
1958                 if (r)
1959                         return r;
1960         }
1961
1962         if (adev->flags & AMD_IS_APU) {
1963                 /* TODO: double check the cp_table_size for RV */
1964                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1965                 r = amdgpu_gfx_rlc_init_cpt(adev);
1966                 if (r)
1967                         return r;
1968         }
1969
1970         switch (adev->ip_versions[GC_HWIP][0]) {
1971         case IP_VERSION(9, 2, 2):
1972         case IP_VERSION(9, 1, 0):
1973                 gfx_v9_0_init_lbpw(adev);
1974                 break;
1975         case IP_VERSION(9, 4, 0):
1976                 gfx_v9_4_init_lbpw(adev);
1977                 break;
1978         default:
1979                 break;
1980         }
1981
1982         /* init spm vmid with 0xf */
1983         if (adev->gfx.rlc.funcs->update_spm_vmid)
1984                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1985
1986         return 0;
1987 }
1988
1989 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1990 {
1991         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1992         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1993 }
1994
1995 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1996 {
1997         int r;
1998         u32 *hpd;
1999         const __le32 *fw_data;
2000         unsigned fw_size;
2001         u32 *fw;
2002         size_t mec_hpd_size;
2003
2004         const struct gfx_firmware_header_v1_0 *mec_hdr;
2005
2006         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2007
2008         /* take ownership of the relevant compute queues */
2009         amdgpu_gfx_compute_queue_acquire(adev);
2010         mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
2011         if (mec_hpd_size) {
2012                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
2013                                               AMDGPU_GEM_DOMAIN_VRAM,
2014                                               &adev->gfx.mec.hpd_eop_obj,
2015                                               &adev->gfx.mec.hpd_eop_gpu_addr,
2016                                               (void **)&hpd);
2017                 if (r) {
2018                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
2019                         gfx_v9_0_mec_fini(adev);
2020                         return r;
2021                 }
2022
2023                 memset(hpd, 0, mec_hpd_size);
2024
2025                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
2026                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
2027         }
2028
2029         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
2030
2031         fw_data = (const __le32 *)
2032                 (adev->gfx.mec_fw->data +
2033                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
2034         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
2035
2036         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
2037                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2038                                       &adev->gfx.mec.mec_fw_obj,
2039                                       &adev->gfx.mec.mec_fw_gpu_addr,
2040                                       (void **)&fw);
2041         if (r) {
2042                 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
2043                 gfx_v9_0_mec_fini(adev);
2044                 return r;
2045         }
2046
2047         memcpy(fw, fw_data, fw_size);
2048
2049         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
2050         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
2051
2052         return 0;
2053 }
2054
2055 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
2056 {
2057         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2058                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2059                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2060                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
2061                 (SQ_IND_INDEX__FORCE_READ_MASK));
2062         return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2063 }
2064
2065 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
2066                            uint32_t wave, uint32_t thread,
2067                            uint32_t regno, uint32_t num, uint32_t *out)
2068 {
2069         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2070                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2071                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2072                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
2073                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
2074                 (SQ_IND_INDEX__FORCE_READ_MASK) |
2075                 (SQ_IND_INDEX__AUTO_INCR_MASK));
2076         while (num--)
2077                 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2078 }
2079
2080 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
2081 {
2082         /* type 1 wave data */
2083         dst[(*no_fields)++] = 1;
2084         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
2085         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
2086         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
2087         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
2088         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
2089         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
2090         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
2091         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
2092         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
2093         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
2094         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
2095         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
2096         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
2097         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
2098         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
2099 }
2100
2101 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
2102                                      uint32_t wave, uint32_t start,
2103                                      uint32_t size, uint32_t *dst)
2104 {
2105         wave_read_regs(
2106                 adev, simd, wave, 0,
2107                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
2108 }
2109
2110 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
2111                                      uint32_t wave, uint32_t thread,
2112                                      uint32_t start, uint32_t size,
2113                                      uint32_t *dst)
2114 {
2115         wave_read_regs(
2116                 adev, simd, wave, thread,
2117                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
2118 }
2119
2120 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
2121                                   u32 me, u32 pipe, u32 q, u32 vm)
2122 {
2123         soc15_grbm_select(adev, me, pipe, q, vm);
2124 }
2125
2126 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2127         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2128         .select_se_sh = &gfx_v9_0_select_se_sh,
2129         .read_wave_data = &gfx_v9_0_read_wave_data,
2130         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2131         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2132         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2133 };
2134
2135 static const struct amdgpu_gfx_ras_funcs gfx_v9_0_ras_funcs = {
2136         .ras_late_init = amdgpu_gfx_ras_late_init,
2137         .ras_fini = amdgpu_gfx_ras_fini,
2138         .ras_error_inject = &gfx_v9_0_ras_error_inject,
2139         .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2140         .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2141 };
2142
2143 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2144 {
2145         u32 gb_addr_config;
2146         int err;
2147
2148         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
2149
2150         switch (adev->ip_versions[GC_HWIP][0]) {
2151         case IP_VERSION(9, 0, 1):
2152                 adev->gfx.config.max_hw_contexts = 8;
2153                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2154                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2155                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2156                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2157                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2158                 break;
2159         case IP_VERSION(9, 2, 1):
2160                 adev->gfx.config.max_hw_contexts = 8;
2161                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2162                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2163                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2164                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2165                 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2166                 DRM_INFO("fix gfx.config for vega12\n");
2167                 break;
2168         case IP_VERSION(9, 4, 0):
2169                 adev->gfx.ras_funcs = &gfx_v9_0_ras_funcs;
2170                 adev->gfx.config.max_hw_contexts = 8;
2171                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2172                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2173                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2174                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2175                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2176                 gb_addr_config &= ~0xf3e777ff;
2177                 gb_addr_config |= 0x22014042;
2178                 /* check vbios table if gpu info is not available */
2179                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2180                 if (err)
2181                         return err;
2182                 break;
2183         case IP_VERSION(9, 2, 2):
2184         case IP_VERSION(9, 1, 0):
2185                 adev->gfx.config.max_hw_contexts = 8;
2186                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2187                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2188                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2189                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2190                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2191                         gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2192                 else
2193                         gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2194                 break;
2195         case IP_VERSION(9, 4, 1):
2196                 adev->gfx.ras_funcs = &gfx_v9_4_ras_funcs;
2197                 adev->gfx.config.max_hw_contexts = 8;
2198                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2199                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2200                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2201                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2202                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2203                 gb_addr_config &= ~0xf3e777ff;
2204                 gb_addr_config |= 0x22014042;
2205                 break;
2206         case IP_VERSION(9, 3, 0):
2207                 adev->gfx.config.max_hw_contexts = 8;
2208                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2209                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2210                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2211                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2212                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2213                 gb_addr_config &= ~0xf3e777ff;
2214                 gb_addr_config |= 0x22010042;
2215                 break;
2216         case IP_VERSION(9, 4, 2):
2217                 adev->gfx.ras_funcs = &gfx_v9_4_2_ras_funcs;
2218                 adev->gfx.config.max_hw_contexts = 8;
2219                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2220                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2221                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2222                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2223                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2224                 gb_addr_config &= ~0xf3e777ff;
2225                 gb_addr_config |= 0x22014042;
2226                 /* check vbios table if gpu info is not available */
2227                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2228                 if (err)
2229                         return err;
2230                 break;
2231         default:
2232                 BUG();
2233                 break;
2234         }
2235
2236         adev->gfx.config.gb_addr_config = gb_addr_config;
2237
2238         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2239                         REG_GET_FIELD(
2240                                         adev->gfx.config.gb_addr_config,
2241                                         GB_ADDR_CONFIG,
2242                                         NUM_PIPES);
2243
2244         adev->gfx.config.max_tile_pipes =
2245                 adev->gfx.config.gb_addr_config_fields.num_pipes;
2246
2247         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2248                         REG_GET_FIELD(
2249                                         adev->gfx.config.gb_addr_config,
2250                                         GB_ADDR_CONFIG,
2251                                         NUM_BANKS);
2252         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2253                         REG_GET_FIELD(
2254                                         adev->gfx.config.gb_addr_config,
2255                                         GB_ADDR_CONFIG,
2256                                         MAX_COMPRESSED_FRAGS);
2257         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2258                         REG_GET_FIELD(
2259                                         adev->gfx.config.gb_addr_config,
2260                                         GB_ADDR_CONFIG,
2261                                         NUM_RB_PER_SE);
2262         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2263                         REG_GET_FIELD(
2264                                         adev->gfx.config.gb_addr_config,
2265                                         GB_ADDR_CONFIG,
2266                                         NUM_SHADER_ENGINES);
2267         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2268                         REG_GET_FIELD(
2269                                         adev->gfx.config.gb_addr_config,
2270                                         GB_ADDR_CONFIG,
2271                                         PIPE_INTERLEAVE_SIZE));
2272
2273         return 0;
2274 }
2275
2276 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2277                                       int mec, int pipe, int queue)
2278 {
2279         unsigned irq_type;
2280         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2281         unsigned int hw_prio;
2282
2283         ring = &adev->gfx.compute_ring[ring_id];
2284
2285         /* mec0 is me1 */
2286         ring->me = mec + 1;
2287         ring->pipe = pipe;
2288         ring->queue = queue;
2289
2290         ring->ring_obj = NULL;
2291         ring->use_doorbell = true;
2292         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2293         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2294                                 + (ring_id * GFX9_MEC_HPD_SIZE);
2295         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2296
2297         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2298                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2299                 + ring->pipe;
2300         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2301                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
2302         /* type-2 packets are deprecated on MEC, use type-3 instead */
2303         return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2304                                 hw_prio, NULL);
2305 }
2306
2307 static int gfx_v9_0_sw_init(void *handle)
2308 {
2309         int i, j, k, r, ring_id;
2310         struct amdgpu_ring *ring;
2311         struct amdgpu_kiq *kiq;
2312         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2313
2314         switch (adev->ip_versions[GC_HWIP][0]) {
2315         case IP_VERSION(9, 0, 1):
2316         case IP_VERSION(9, 2, 1):
2317         case IP_VERSION(9, 4, 0):
2318         case IP_VERSION(9, 2, 2):
2319         case IP_VERSION(9, 1, 0):
2320         case IP_VERSION(9, 4, 1):
2321         case IP_VERSION(9, 3, 0):
2322         case IP_VERSION(9, 4, 2):
2323                 adev->gfx.mec.num_mec = 2;
2324                 break;
2325         default:
2326                 adev->gfx.mec.num_mec = 1;
2327                 break;
2328         }
2329
2330         adev->gfx.mec.num_pipe_per_mec = 4;
2331         adev->gfx.mec.num_queue_per_pipe = 8;
2332
2333         /* EOP Event */
2334         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2335         if (r)
2336                 return r;
2337
2338         /* Privileged reg */
2339         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2340                               &adev->gfx.priv_reg_irq);
2341         if (r)
2342                 return r;
2343
2344         /* Privileged inst */
2345         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2346                               &adev->gfx.priv_inst_irq);
2347         if (r)
2348                 return r;
2349
2350         /* ECC error */
2351         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2352                               &adev->gfx.cp_ecc_error_irq);
2353         if (r)
2354                 return r;
2355
2356         /* FUE error */
2357         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2358                               &adev->gfx.cp_ecc_error_irq);
2359         if (r)
2360                 return r;
2361
2362         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2363
2364         gfx_v9_0_scratch_init(adev);
2365
2366         r = gfx_v9_0_init_microcode(adev);
2367         if (r) {
2368                 DRM_ERROR("Failed to load gfx firmware!\n");
2369                 return r;
2370         }
2371
2372         r = adev->gfx.rlc.funcs->init(adev);
2373         if (r) {
2374                 DRM_ERROR("Failed to init rlc BOs!\n");
2375                 return r;
2376         }
2377
2378         r = gfx_v9_0_mec_init(adev);
2379         if (r) {
2380                 DRM_ERROR("Failed to init MEC BOs!\n");
2381                 return r;
2382         }
2383
2384         /* set up the gfx ring */
2385         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2386                 ring = &adev->gfx.gfx_ring[i];
2387                 ring->ring_obj = NULL;
2388                 if (!i)
2389                         sprintf(ring->name, "gfx");
2390                 else
2391                         sprintf(ring->name, "gfx_%d", i);
2392                 ring->use_doorbell = true;
2393                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2394                 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2395                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2396                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
2397                 if (r)
2398                         return r;
2399         }
2400
2401         /* set up the compute queues - allocate horizontally across pipes */
2402         ring_id = 0;
2403         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2404                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2405                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2406                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2407                                         continue;
2408
2409                                 r = gfx_v9_0_compute_ring_init(adev,
2410                                                                ring_id,
2411                                                                i, k, j);
2412                                 if (r)
2413                                         return r;
2414
2415                                 ring_id++;
2416                         }
2417                 }
2418         }
2419
2420         r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2421         if (r) {
2422                 DRM_ERROR("Failed to init KIQ BOs!\n");
2423                 return r;
2424         }
2425
2426         kiq = &adev->gfx.kiq;
2427         r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2428         if (r)
2429                 return r;
2430
2431         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2432         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2433         if (r)
2434                 return r;
2435
2436         adev->gfx.ce_ram_size = 0x8000;
2437
2438         r = gfx_v9_0_gpu_early_init(adev);
2439         if (r)
2440                 return r;
2441
2442         return 0;
2443 }
2444
2445
2446 static int gfx_v9_0_sw_fini(void *handle)
2447 {
2448         int i;
2449         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2450
2451         if (adev->gfx.ras_funcs &&
2452             adev->gfx.ras_funcs->ras_fini)
2453                 adev->gfx.ras_funcs->ras_fini(adev);
2454
2455         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2456                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2457         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2458                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2459
2460         amdgpu_gfx_mqd_sw_fini(adev);
2461         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2462         amdgpu_gfx_kiq_fini(adev);
2463
2464         gfx_v9_0_mec_fini(adev);
2465         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2466                                 &adev->gfx.rlc.clear_state_gpu_addr,
2467                                 (void **)&adev->gfx.rlc.cs_ptr);
2468         if (adev->flags & AMD_IS_APU) {
2469                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2470                                 &adev->gfx.rlc.cp_table_gpu_addr,
2471                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2472         }
2473         gfx_v9_0_free_microcode(adev);
2474
2475         return 0;
2476 }
2477
2478
2479 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2480 {
2481         /* TODO */
2482 }
2483
2484 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2485                            u32 instance)
2486 {
2487         u32 data;
2488
2489         if (instance == 0xffffffff)
2490                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2491         else
2492                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2493
2494         if (se_num == 0xffffffff)
2495                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2496         else
2497                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2498
2499         if (sh_num == 0xffffffff)
2500                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2501         else
2502                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2503
2504         WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2505 }
2506
2507 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2508 {
2509         u32 data, mask;
2510
2511         data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2512         data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2513
2514         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2515         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2516
2517         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2518                                          adev->gfx.config.max_sh_per_se);
2519
2520         return (~data) & mask;
2521 }
2522
2523 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2524 {
2525         int i, j;
2526         u32 data;
2527         u32 active_rbs = 0;
2528         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2529                                         adev->gfx.config.max_sh_per_se;
2530
2531         mutex_lock(&adev->grbm_idx_mutex);
2532         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2533                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2534                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2535                         data = gfx_v9_0_get_rb_active_bitmap(adev);
2536                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2537                                                rb_bitmap_width_per_sh);
2538                 }
2539         }
2540         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2541         mutex_unlock(&adev->grbm_idx_mutex);
2542
2543         adev->gfx.config.backend_enable_mask = active_rbs;
2544         adev->gfx.config.num_rbs = hweight32(active_rbs);
2545 }
2546
2547 #define DEFAULT_SH_MEM_BASES    (0x6000)
2548 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2549 {
2550         int i;
2551         uint32_t sh_mem_config;
2552         uint32_t sh_mem_bases;
2553
2554         /*
2555          * Configure apertures:
2556          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
2557          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
2558          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
2559          */
2560         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2561
2562         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2563                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2564                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2565
2566         mutex_lock(&adev->srbm_mutex);
2567         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2568                 soc15_grbm_select(adev, 0, 0, 0, i);
2569                 /* CP and shaders */
2570                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2571                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2572         }
2573         soc15_grbm_select(adev, 0, 0, 0, 0);
2574         mutex_unlock(&adev->srbm_mutex);
2575
2576         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2577            acccess. These should be enabled by FW for target VMIDs. */
2578         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2579                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2580                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2581                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2582                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2583         }
2584 }
2585
2586 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2587 {
2588         int vmid;
2589
2590         /*
2591          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2592          * access. Compute VMIDs should be enabled by FW for target VMIDs,
2593          * the driver can enable them for graphics. VMID0 should maintain
2594          * access so that HWS firmware can save/restore entries.
2595          */
2596         for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2597                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2598                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2599                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2600                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2601         }
2602 }
2603
2604 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2605 {
2606         uint32_t tmp;
2607
2608         switch (adev->ip_versions[GC_HWIP][0]) {
2609         case IP_VERSION(9, 4, 1):
2610                 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2611                 tmp = REG_SET_FIELD(tmp, SQ_CONFIG,
2612                                         DISABLE_BARRIER_WAITCNT, 1);
2613                 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2614                 break;
2615         default:
2616                 break;
2617         }
2618 }
2619
2620 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2621 {
2622         u32 tmp;
2623         int i;
2624
2625         WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2626
2627         gfx_v9_0_tiling_mode_table_init(adev);
2628
2629         gfx_v9_0_setup_rb(adev);
2630         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2631         adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2632
2633         /* XXX SH_MEM regs */
2634         /* where to put LDS, scratch, GPUVM in FSA64 space */
2635         mutex_lock(&adev->srbm_mutex);
2636         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2637                 soc15_grbm_select(adev, 0, 0, 0, i);
2638                 /* CP and shaders */
2639                 if (i == 0) {
2640                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2641                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2642                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2643                                             !!adev->gmc.noretry);
2644                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2645                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2646                 } else {
2647                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2648                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2649                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2650                                             !!adev->gmc.noretry);
2651                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2652                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2653                                 (adev->gmc.private_aperture_start >> 48));
2654                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2655                                 (adev->gmc.shared_aperture_start >> 48));
2656                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2657                 }
2658         }
2659         soc15_grbm_select(adev, 0, 0, 0, 0);
2660
2661         mutex_unlock(&adev->srbm_mutex);
2662
2663         gfx_v9_0_init_compute_vmid(adev);
2664         gfx_v9_0_init_gds_vmid(adev);
2665         gfx_v9_0_init_sq_config(adev);
2666 }
2667
2668 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2669 {
2670         u32 i, j, k;
2671         u32 mask;
2672
2673         mutex_lock(&adev->grbm_idx_mutex);
2674         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2675                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2676                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2677                         for (k = 0; k < adev->usec_timeout; k++) {
2678                                 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2679                                         break;
2680                                 udelay(1);
2681                         }
2682                         if (k == adev->usec_timeout) {
2683                                 gfx_v9_0_select_se_sh(adev, 0xffffffff,
2684                                                       0xffffffff, 0xffffffff);
2685                                 mutex_unlock(&adev->grbm_idx_mutex);
2686                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2687                                          i, j);
2688                                 return;
2689                         }
2690                 }
2691         }
2692         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2693         mutex_unlock(&adev->grbm_idx_mutex);
2694
2695         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2696                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2697                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2698                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2699         for (k = 0; k < adev->usec_timeout; k++) {
2700                 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2701                         break;
2702                 udelay(1);
2703         }
2704 }
2705
2706 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2707                                                bool enable)
2708 {
2709         u32 tmp;
2710
2711         /* These interrupts should be enabled to drive DS clock */
2712
2713         tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2714
2715         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2716         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2717         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2718         if(adev->gfx.num_gfx_rings)
2719                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2720
2721         WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2722 }
2723
2724 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2725 {
2726         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2727         /* csib */
2728         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2729                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
2730         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2731                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2732         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2733                         adev->gfx.rlc.clear_state_size);
2734 }
2735
2736 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2737                                 int indirect_offset,
2738                                 int list_size,
2739                                 int *unique_indirect_regs,
2740                                 int unique_indirect_reg_count,
2741                                 int *indirect_start_offsets,
2742                                 int *indirect_start_offsets_count,
2743                                 int max_start_offsets_count)
2744 {
2745         int idx;
2746
2747         for (; indirect_offset < list_size; indirect_offset++) {
2748                 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2749                 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2750                 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2751
2752                 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2753                         indirect_offset += 2;
2754
2755                         /* look for the matching indice */
2756                         for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2757                                 if (unique_indirect_regs[idx] ==
2758                                         register_list_format[indirect_offset] ||
2759                                         !unique_indirect_regs[idx])
2760                                         break;
2761                         }
2762
2763                         BUG_ON(idx >= unique_indirect_reg_count);
2764
2765                         if (!unique_indirect_regs[idx])
2766                                 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2767
2768                         indirect_offset++;
2769                 }
2770         }
2771 }
2772
2773 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2774 {
2775         int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2776         int unique_indirect_reg_count = 0;
2777
2778         int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2779         int indirect_start_offsets_count = 0;
2780
2781         int list_size = 0;
2782         int i = 0, j = 0;
2783         u32 tmp = 0;
2784
2785         u32 *register_list_format =
2786                 kmemdup(adev->gfx.rlc.register_list_format,
2787                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2788         if (!register_list_format)
2789                 return -ENOMEM;
2790
2791         /* setup unique_indirect_regs array and indirect_start_offsets array */
2792         unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2793         gfx_v9_1_parse_ind_reg_list(register_list_format,
2794                                     adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2795                                     adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2796                                     unique_indirect_regs,
2797                                     unique_indirect_reg_count,
2798                                     indirect_start_offsets,
2799                                     &indirect_start_offsets_count,
2800                                     ARRAY_SIZE(indirect_start_offsets));
2801
2802         /* enable auto inc in case it is disabled */
2803         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2804         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2805         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2806
2807         /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2808         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2809                 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2810         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2811                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2812                         adev->gfx.rlc.register_restore[i]);
2813
2814         /* load indirect register */
2815         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2816                 adev->gfx.rlc.reg_list_format_start);
2817
2818         /* direct register portion */
2819         for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2820                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2821                         register_list_format[i]);
2822
2823         /* indirect register portion */
2824         while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2825                 if (register_list_format[i] == 0xFFFFFFFF) {
2826                         WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2827                         continue;
2828                 }
2829
2830                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2831                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2832
2833                 for (j = 0; j < unique_indirect_reg_count; j++) {
2834                         if (register_list_format[i] == unique_indirect_regs[j]) {
2835                                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2836                                 break;
2837                         }
2838                 }
2839
2840                 BUG_ON(j >= unique_indirect_reg_count);
2841
2842                 i++;
2843         }
2844
2845         /* set save/restore list size */
2846         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2847         list_size = list_size >> 1;
2848         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2849                 adev->gfx.rlc.reg_restore_list_size);
2850         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2851
2852         /* write the starting offsets to RLC scratch ram */
2853         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2854                 adev->gfx.rlc.starting_offsets_start);
2855         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2856                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2857                        indirect_start_offsets[i]);
2858
2859         /* load unique indirect regs*/
2860         for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2861                 if (unique_indirect_regs[i] != 0) {
2862                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2863                                + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2864                                unique_indirect_regs[i] & 0x3FFFF);
2865
2866                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2867                                + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2868                                unique_indirect_regs[i] >> 20);
2869                 }
2870         }
2871
2872         kfree(register_list_format);
2873         return 0;
2874 }
2875
2876 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2877 {
2878         WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2879 }
2880
2881 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2882                                              bool enable)
2883 {
2884         uint32_t data = 0;
2885         uint32_t default_data = 0;
2886
2887         default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2888         if (enable) {
2889                 /* enable GFXIP control over CGPG */
2890                 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2891                 if(default_data != data)
2892                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2893
2894                 /* update status */
2895                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2896                 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2897                 if(default_data != data)
2898                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2899         } else {
2900                 /* restore GFXIP control over GCPG */
2901                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2902                 if(default_data != data)
2903                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2904         }
2905 }
2906
2907 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2908 {
2909         uint32_t data = 0;
2910
2911         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2912                               AMD_PG_SUPPORT_GFX_SMG |
2913                               AMD_PG_SUPPORT_GFX_DMG)) {
2914                 /* init IDLE_POLL_COUNT = 60 */
2915                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2916                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2917                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2918                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2919
2920                 /* init RLC PG Delay */
2921                 data = 0;
2922                 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2923                 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2924                 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2925                 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2926                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2927
2928                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2929                 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2930                 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2931                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2932
2933                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2934                 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2935                 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2936                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2937
2938                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2939                 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2940
2941                 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2942                 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2943                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2944                 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 3, 0))
2945                         pwr_10_0_gfxip_control_over_cgpg(adev, true);
2946         }
2947 }
2948
2949 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2950                                                 bool enable)
2951 {
2952         uint32_t data = 0;
2953         uint32_t default_data = 0;
2954
2955         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2956         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2957                              SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2958                              enable ? 1 : 0);
2959         if (default_data != data)
2960                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2961 }
2962
2963 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2964                                                 bool enable)
2965 {
2966         uint32_t data = 0;
2967         uint32_t default_data = 0;
2968
2969         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2970         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2971                              SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2972                              enable ? 1 : 0);
2973         if(default_data != data)
2974                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2975 }
2976
2977 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2978                                         bool enable)
2979 {
2980         uint32_t data = 0;
2981         uint32_t default_data = 0;
2982
2983         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2984         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2985                              CP_PG_DISABLE,
2986                              enable ? 0 : 1);
2987         if(default_data != data)
2988                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2989 }
2990
2991 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2992                                                 bool enable)
2993 {
2994         uint32_t data, default_data;
2995
2996         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2997         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2998                              GFX_POWER_GATING_ENABLE,
2999                              enable ? 1 : 0);
3000         if(default_data != data)
3001                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3002 }
3003
3004 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
3005                                                 bool enable)
3006 {
3007         uint32_t data, default_data;
3008
3009         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3010         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3011                              GFX_PIPELINE_PG_ENABLE,
3012                              enable ? 1 : 0);
3013         if(default_data != data)
3014                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3015
3016         if (!enable)
3017                 /* read any GFX register to wake up GFX */
3018                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
3019 }
3020
3021 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3022                                                        bool enable)
3023 {
3024         uint32_t data, default_data;
3025
3026         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3027         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3028                              STATIC_PER_CU_PG_ENABLE,
3029                              enable ? 1 : 0);
3030         if(default_data != data)
3031                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3032 }
3033
3034 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3035                                                 bool enable)
3036 {
3037         uint32_t data, default_data;
3038
3039         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3040         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3041                              DYN_PER_CU_PG_ENABLE,
3042                              enable ? 1 : 0);
3043         if(default_data != data)
3044                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3045 }
3046
3047 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3048 {
3049         gfx_v9_0_init_csb(adev);
3050
3051         /*
3052          * Rlc save restore list is workable since v2_1.
3053          * And it's needed by gfxoff feature.
3054          */
3055         if (adev->gfx.rlc.is_rlc_v2_1) {
3056                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 2, 1) ||
3057                     (adev->apu_flags & AMD_APU_IS_RAVEN2))
3058                         gfx_v9_1_init_rlc_save_restore_list(adev);
3059                 gfx_v9_0_enable_save_restore_machine(adev);
3060         }
3061
3062         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3063                               AMD_PG_SUPPORT_GFX_SMG |
3064                               AMD_PG_SUPPORT_GFX_DMG |
3065                               AMD_PG_SUPPORT_CP |
3066                               AMD_PG_SUPPORT_GDS |
3067                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
3068                 WREG32(mmRLC_JUMP_TABLE_RESTORE,
3069                        adev->gfx.rlc.cp_table_gpu_addr >> 8);
3070                 gfx_v9_0_init_gfx_power_gating(adev);
3071         }
3072 }
3073
3074 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3075 {
3076         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3077         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3078         gfx_v9_0_wait_for_rlc_serdes(adev);
3079 }
3080
3081 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3082 {
3083         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3084         udelay(50);
3085         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3086         udelay(50);
3087 }
3088
3089 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3090 {
3091 #ifdef AMDGPU_RLC_DEBUG_RETRY
3092         u32 rlc_ucode_ver;
3093 #endif
3094
3095         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3096         udelay(50);
3097
3098         /* carrizo do enable cp interrupt after cp inited */
3099         if (!(adev->flags & AMD_IS_APU)) {
3100                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3101                 udelay(50);
3102         }
3103
3104 #ifdef AMDGPU_RLC_DEBUG_RETRY
3105         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3106         rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3107         if(rlc_ucode_ver == 0x108) {
3108                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3109                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3110                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3111                  * default is 0x9C4 to create a 100us interval */
3112                 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3113                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3114                  * to disable the page fault retry interrupts, default is
3115                  * 0x100 (256) */
3116                 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3117         }
3118 #endif
3119 }
3120
3121 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3122 {
3123         const struct rlc_firmware_header_v2_0 *hdr;
3124         const __le32 *fw_data;
3125         unsigned i, fw_size;
3126
3127         if (!adev->gfx.rlc_fw)
3128                 return -EINVAL;
3129
3130         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3131         amdgpu_ucode_print_rlc_hdr(&hdr->header);
3132
3133         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3134                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3135         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3136
3137         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3138                         RLCG_UCODE_LOADING_START_ADDRESS);
3139         for (i = 0; i < fw_size; i++)
3140                 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3141         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3142
3143         return 0;
3144 }
3145
3146 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3147 {
3148         int r;
3149
3150         if (amdgpu_sriov_vf(adev)) {
3151                 gfx_v9_0_init_csb(adev);
3152                 return 0;
3153         }
3154
3155         adev->gfx.rlc.funcs->stop(adev);
3156
3157         /* disable CG */
3158         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3159
3160         gfx_v9_0_init_pg(adev);
3161
3162         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3163                 /* legacy rlc firmware loading */
3164                 r = gfx_v9_0_rlc_load_microcode(adev);
3165                 if (r)
3166                         return r;
3167         }
3168
3169         switch (adev->ip_versions[GC_HWIP][0]) {
3170         case IP_VERSION(9, 2, 2):
3171         case IP_VERSION(9, 1, 0):
3172                 if (amdgpu_lbpw == 0)
3173                         gfx_v9_0_enable_lbpw(adev, false);
3174                 else
3175                         gfx_v9_0_enable_lbpw(adev, true);
3176                 break;
3177         case IP_VERSION(9, 4, 0):
3178                 if (amdgpu_lbpw > 0)
3179                         gfx_v9_0_enable_lbpw(adev, true);
3180                 else
3181                         gfx_v9_0_enable_lbpw(adev, false);
3182                 break;
3183         default:
3184                 break;
3185         }
3186
3187         adev->gfx.rlc.funcs->start(adev);
3188
3189         return 0;
3190 }
3191
3192 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3193 {
3194         u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3195
3196         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3197         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3198         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3199         WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3200         udelay(50);
3201 }
3202
3203 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3204 {
3205         const struct gfx_firmware_header_v1_0 *pfp_hdr;
3206         const struct gfx_firmware_header_v1_0 *ce_hdr;
3207         const struct gfx_firmware_header_v1_0 *me_hdr;
3208         const __le32 *fw_data;
3209         unsigned i, fw_size;
3210
3211         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3212                 return -EINVAL;
3213
3214         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3215                 adev->gfx.pfp_fw->data;
3216         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3217                 adev->gfx.ce_fw->data;
3218         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3219                 adev->gfx.me_fw->data;
3220
3221         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3222         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3223         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3224
3225         gfx_v9_0_cp_gfx_enable(adev, false);
3226
3227         /* PFP */
3228         fw_data = (const __le32 *)
3229                 (adev->gfx.pfp_fw->data +
3230                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3231         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3232         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3233         for (i = 0; i < fw_size; i++)
3234                 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3235         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3236
3237         /* CE */
3238         fw_data = (const __le32 *)
3239                 (adev->gfx.ce_fw->data +
3240                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3241         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3242         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3243         for (i = 0; i < fw_size; i++)
3244                 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3245         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3246
3247         /* ME */
3248         fw_data = (const __le32 *)
3249                 (adev->gfx.me_fw->data +
3250                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3251         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3252         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3253         for (i = 0; i < fw_size; i++)
3254                 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3255         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3256
3257         return 0;
3258 }
3259
3260 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3261 {
3262         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3263         const struct cs_section_def *sect = NULL;
3264         const struct cs_extent_def *ext = NULL;
3265         int r, i, tmp;
3266
3267         /* init the CP */
3268         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3269         WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3270
3271         gfx_v9_0_cp_gfx_enable(adev, true);
3272
3273         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3274         if (r) {
3275                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3276                 return r;
3277         }
3278
3279         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3280         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3281
3282         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3283         amdgpu_ring_write(ring, 0x80000000);
3284         amdgpu_ring_write(ring, 0x80000000);
3285
3286         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3287                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3288                         if (sect->id == SECT_CONTEXT) {
3289                                 amdgpu_ring_write(ring,
3290                                        PACKET3(PACKET3_SET_CONTEXT_REG,
3291                                                ext->reg_count));
3292                                 amdgpu_ring_write(ring,
3293                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3294                                 for (i = 0; i < ext->reg_count; i++)
3295                                         amdgpu_ring_write(ring, ext->extent[i]);
3296                         }
3297                 }
3298         }
3299
3300         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3301         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3302
3303         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3304         amdgpu_ring_write(ring, 0);
3305
3306         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3307         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3308         amdgpu_ring_write(ring, 0x8000);
3309         amdgpu_ring_write(ring, 0x8000);
3310
3311         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3312         tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3313                 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3314         amdgpu_ring_write(ring, tmp);
3315         amdgpu_ring_write(ring, 0);
3316
3317         amdgpu_ring_commit(ring);
3318
3319         return 0;
3320 }
3321
3322 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3323 {
3324         struct amdgpu_ring *ring;
3325         u32 tmp;
3326         u32 rb_bufsz;
3327         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3328
3329         /* Set the write pointer delay */
3330         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3331
3332         /* set the RB to use vmid 0 */
3333         WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3334
3335         /* Set ring buffer size */
3336         ring = &adev->gfx.gfx_ring[0];
3337         rb_bufsz = order_base_2(ring->ring_size / 8);
3338         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3339         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3340 #ifdef __BIG_ENDIAN
3341         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3342 #endif
3343         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3344
3345         /* Initialize the ring buffer's write pointers */
3346         ring->wptr = 0;
3347         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3348         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3349
3350         /* set the wb address wether it's enabled or not */
3351         rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3352         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3353         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3354
3355         wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3356         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3357         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3358
3359         mdelay(1);
3360         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3361
3362         rb_addr = ring->gpu_addr >> 8;
3363         WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3364         WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3365
3366         tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3367         if (ring->use_doorbell) {
3368                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3369                                     DOORBELL_OFFSET, ring->doorbell_index);
3370                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3371                                     DOORBELL_EN, 1);
3372         } else {
3373                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3374         }
3375         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3376
3377         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3378                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
3379         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3380
3381         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3382                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3383
3384
3385         /* start the ring */
3386         gfx_v9_0_cp_gfx_start(adev);
3387         ring->sched.ready = true;
3388
3389         return 0;
3390 }
3391
3392 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3393 {
3394         if (enable) {
3395                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3396         } else {
3397                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3398                         (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3399                 adev->gfx.kiq.ring.sched.ready = false;
3400         }
3401         udelay(50);
3402 }
3403
3404 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3405 {
3406         const struct gfx_firmware_header_v1_0 *mec_hdr;
3407         const __le32 *fw_data;
3408         unsigned i;
3409         u32 tmp;
3410
3411         if (!adev->gfx.mec_fw)
3412                 return -EINVAL;
3413
3414         gfx_v9_0_cp_compute_enable(adev, false);
3415
3416         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3417         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3418
3419         fw_data = (const __le32 *)
3420                 (adev->gfx.mec_fw->data +
3421                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3422         tmp = 0;
3423         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3424         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3425         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3426
3427         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3428                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3429         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3430                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3431
3432         /* MEC1 */
3433         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3434                          mec_hdr->jt_offset);
3435         for (i = 0; i < mec_hdr->jt_size; i++)
3436                 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3437                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3438
3439         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3440                         adev->gfx.mec_fw_version);
3441         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3442
3443         return 0;
3444 }
3445
3446 /* KIQ functions */
3447 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3448 {
3449         uint32_t tmp;
3450         struct amdgpu_device *adev = ring->adev;
3451
3452         /* tell RLC which is KIQ queue */
3453         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3454         tmp &= 0xffffff00;
3455         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3456         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3457         tmp |= 0x80;
3458         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3459 }
3460
3461 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3462 {
3463         struct amdgpu_device *adev = ring->adev;
3464
3465         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3466                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3467                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3468                         mqd->cp_hqd_queue_priority =
3469                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3470                 }
3471         }
3472 }
3473
3474 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3475 {
3476         struct amdgpu_device *adev = ring->adev;
3477         struct v9_mqd *mqd = ring->mqd_ptr;
3478         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3479         uint32_t tmp;
3480
3481         mqd->header = 0xC0310800;
3482         mqd->compute_pipelinestat_enable = 0x00000001;
3483         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3484         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3485         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3486         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3487         mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3488         mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3489         mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3490         mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3491         mqd->compute_misc_reserved = 0x00000003;
3492
3493         mqd->dynamic_cu_mask_addr_lo =
3494                 lower_32_bits(ring->mqd_gpu_addr
3495                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3496         mqd->dynamic_cu_mask_addr_hi =
3497                 upper_32_bits(ring->mqd_gpu_addr
3498                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3499
3500         eop_base_addr = ring->eop_gpu_addr >> 8;
3501         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3502         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3503
3504         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3505         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3506         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3507                         (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3508
3509         mqd->cp_hqd_eop_control = tmp;
3510
3511         /* enable doorbell? */
3512         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3513
3514         if (ring->use_doorbell) {
3515                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3516                                     DOORBELL_OFFSET, ring->doorbell_index);
3517                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3518                                     DOORBELL_EN, 1);
3519                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3520                                     DOORBELL_SOURCE, 0);
3521                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3522                                     DOORBELL_HIT, 0);
3523         } else {
3524                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3525                                          DOORBELL_EN, 0);
3526         }
3527
3528         mqd->cp_hqd_pq_doorbell_control = tmp;
3529
3530         /* disable the queue if it's active */
3531         ring->wptr = 0;
3532         mqd->cp_hqd_dequeue_request = 0;
3533         mqd->cp_hqd_pq_rptr = 0;
3534         mqd->cp_hqd_pq_wptr_lo = 0;
3535         mqd->cp_hqd_pq_wptr_hi = 0;
3536
3537         /* set the pointer to the MQD */
3538         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3539         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3540
3541         /* set MQD vmid to 0 */
3542         tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3543         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3544         mqd->cp_mqd_control = tmp;
3545
3546         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3547         hqd_gpu_addr = ring->gpu_addr >> 8;
3548         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3549         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3550
3551         /* set up the HQD, this is similar to CP_RB0_CNTL */
3552         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3553         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3554                             (order_base_2(ring->ring_size / 4) - 1));
3555         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3556                         ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3557 #ifdef __BIG_ENDIAN
3558         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3559 #endif
3560         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3561         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3562         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3563         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3564         mqd->cp_hqd_pq_control = tmp;
3565
3566         /* set the wb address whether it's enabled or not */
3567         wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3568         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3569         mqd->cp_hqd_pq_rptr_report_addr_hi =
3570                 upper_32_bits(wb_gpu_addr) & 0xffff;
3571
3572         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3573         wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3574         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3575         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3576
3577         tmp = 0;
3578         /* enable the doorbell if requested */
3579         if (ring->use_doorbell) {
3580                 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3581                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3582                                 DOORBELL_OFFSET, ring->doorbell_index);
3583
3584                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3585                                          DOORBELL_EN, 1);
3586                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3587                                          DOORBELL_SOURCE, 0);
3588                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3589                                          DOORBELL_HIT, 0);
3590         }
3591
3592         mqd->cp_hqd_pq_doorbell_control = tmp;
3593
3594         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3595         ring->wptr = 0;
3596         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3597
3598         /* set the vmid for the queue */
3599         mqd->cp_hqd_vmid = 0;
3600
3601         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3602         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3603         mqd->cp_hqd_persistent_state = tmp;
3604
3605         /* set MIN_IB_AVAIL_SIZE */
3606         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3607         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3608         mqd->cp_hqd_ib_control = tmp;
3609
3610         /* set static priority for a queue/ring */
3611         gfx_v9_0_mqd_set_priority(ring, mqd);
3612         mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
3613
3614         /* map_queues packet doesn't need activate the queue,
3615          * so only kiq need set this field.
3616          */
3617         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3618                 mqd->cp_hqd_active = 1;
3619
3620         return 0;
3621 }
3622
3623 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3624 {
3625         struct amdgpu_device *adev = ring->adev;
3626         struct v9_mqd *mqd = ring->mqd_ptr;
3627         int j;
3628
3629         /* disable wptr polling */
3630         WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3631
3632         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3633                mqd->cp_hqd_eop_base_addr_lo);
3634         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3635                mqd->cp_hqd_eop_base_addr_hi);
3636
3637         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3638         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3639                mqd->cp_hqd_eop_control);
3640
3641         /* enable doorbell? */
3642         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3643                mqd->cp_hqd_pq_doorbell_control);
3644
3645         /* disable the queue if it's active */
3646         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3647                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3648                 for (j = 0; j < adev->usec_timeout; j++) {
3649                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3650                                 break;
3651                         udelay(1);
3652                 }
3653                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3654                        mqd->cp_hqd_dequeue_request);
3655                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3656                        mqd->cp_hqd_pq_rptr);
3657                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3658                        mqd->cp_hqd_pq_wptr_lo);
3659                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3660                        mqd->cp_hqd_pq_wptr_hi);
3661         }
3662
3663         /* set the pointer to the MQD */
3664         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3665                mqd->cp_mqd_base_addr_lo);
3666         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3667                mqd->cp_mqd_base_addr_hi);
3668
3669         /* set MQD vmid to 0 */
3670         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3671                mqd->cp_mqd_control);
3672
3673         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3674         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3675                mqd->cp_hqd_pq_base_lo);
3676         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3677                mqd->cp_hqd_pq_base_hi);
3678
3679         /* set up the HQD, this is similar to CP_RB0_CNTL */
3680         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3681                mqd->cp_hqd_pq_control);
3682
3683         /* set the wb address whether it's enabled or not */
3684         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3685                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3686         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3687                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3688
3689         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3690         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3691                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3692         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3693                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3694
3695         /* enable the doorbell if requested */
3696         if (ring->use_doorbell) {
3697                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3698                                         (adev->doorbell_index.kiq * 2) << 2);
3699                 /* If GC has entered CGPG, ringing doorbell > first page
3700                  * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3701                  * workaround this issue. And this change has to align with firmware
3702                  * update.
3703                  */
3704                 if (check_if_enlarge_doorbell_range(adev))
3705                         WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3706                                         (adev->doorbell.size - 4));
3707                 else
3708                         WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3709                                         (adev->doorbell_index.userqueue_end * 2) << 2);
3710         }
3711
3712         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3713                mqd->cp_hqd_pq_doorbell_control);
3714
3715         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3716         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3717                mqd->cp_hqd_pq_wptr_lo);
3718         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3719                mqd->cp_hqd_pq_wptr_hi);
3720
3721         /* set the vmid for the queue */
3722         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3723
3724         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3725                mqd->cp_hqd_persistent_state);
3726
3727         /* activate the queue */
3728         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3729                mqd->cp_hqd_active);
3730
3731         if (ring->use_doorbell)
3732                 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3733
3734         return 0;
3735 }
3736
3737 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3738 {
3739         struct amdgpu_device *adev = ring->adev;
3740         int j;
3741
3742         /* disable the queue if it's active */
3743         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3744
3745                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3746
3747                 for (j = 0; j < adev->usec_timeout; j++) {
3748                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3749                                 break;
3750                         udelay(1);
3751                 }
3752
3753                 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3754                         DRM_DEBUG("KIQ dequeue request failed.\n");
3755
3756                         /* Manual disable if dequeue request times out */
3757                         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3758                 }
3759
3760                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3761                       0);
3762         }
3763
3764         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3765         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3766         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3767         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3768         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3769         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3770         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3771         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3772
3773         return 0;
3774 }
3775
3776 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3777 {
3778         struct amdgpu_device *adev = ring->adev;
3779         struct v9_mqd *mqd = ring->mqd_ptr;
3780         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3781         struct v9_mqd *tmp_mqd;
3782
3783         gfx_v9_0_kiq_setting(ring);
3784
3785         /* GPU could be in bad state during probe, driver trigger the reset
3786          * after load the SMU, in this case , the mqd is not be initialized.
3787          * driver need to re-init the mqd.
3788          * check mqd->cp_hqd_pq_control since this value should not be 0
3789          */
3790         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3791         if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3792                 /* for GPU_RESET case , reset MQD to a clean status */
3793                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3794                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3795
3796                 /* reset ring buffer */
3797                 ring->wptr = 0;
3798                 amdgpu_ring_clear_ring(ring);
3799
3800                 mutex_lock(&adev->srbm_mutex);
3801                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3802                 gfx_v9_0_kiq_init_register(ring);
3803                 soc15_grbm_select(adev, 0, 0, 0, 0);
3804                 mutex_unlock(&adev->srbm_mutex);
3805         } else {
3806                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3807                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3808                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3809                 mutex_lock(&adev->srbm_mutex);
3810                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3811                 gfx_v9_0_mqd_init(ring);
3812                 gfx_v9_0_kiq_init_register(ring);
3813                 soc15_grbm_select(adev, 0, 0, 0, 0);
3814                 mutex_unlock(&adev->srbm_mutex);
3815
3816                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3817                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3818         }
3819
3820         return 0;
3821 }
3822
3823 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3824 {
3825         struct amdgpu_device *adev = ring->adev;
3826         struct v9_mqd *mqd = ring->mqd_ptr;
3827         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3828         struct v9_mqd *tmp_mqd;
3829
3830         /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3831          * is not be initialized before
3832          */
3833         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3834
3835         if (!tmp_mqd->cp_hqd_pq_control ||
3836             (!amdgpu_in_reset(adev) && !adev->in_suspend)) {
3837                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3838                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3839                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3840                 mutex_lock(&adev->srbm_mutex);
3841                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3842                 gfx_v9_0_mqd_init(ring);
3843                 soc15_grbm_select(adev, 0, 0, 0, 0);
3844                 mutex_unlock(&adev->srbm_mutex);
3845
3846                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3847                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3848         } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
3849                 /* reset MQD to a clean status */
3850                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3851                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3852
3853                 /* reset ring buffer */
3854                 ring->wptr = 0;
3855                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
3856                 amdgpu_ring_clear_ring(ring);
3857         } else {
3858                 amdgpu_ring_clear_ring(ring);
3859         }
3860
3861         return 0;
3862 }
3863
3864 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3865 {
3866         struct amdgpu_ring *ring;
3867         int r;
3868
3869         ring = &adev->gfx.kiq.ring;
3870
3871         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3872         if (unlikely(r != 0))
3873                 return r;
3874
3875         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3876         if (unlikely(r != 0))
3877                 return r;
3878
3879         gfx_v9_0_kiq_init_queue(ring);
3880         amdgpu_bo_kunmap(ring->mqd_obj);
3881         ring->mqd_ptr = NULL;
3882         amdgpu_bo_unreserve(ring->mqd_obj);
3883         ring->sched.ready = true;
3884         return 0;
3885 }
3886
3887 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3888 {
3889         struct amdgpu_ring *ring = NULL;
3890         int r = 0, i;
3891
3892         gfx_v9_0_cp_compute_enable(adev, true);
3893
3894         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3895                 ring = &adev->gfx.compute_ring[i];
3896
3897                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3898                 if (unlikely(r != 0))
3899                         goto done;
3900                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3901                 if (!r) {
3902                         r = gfx_v9_0_kcq_init_queue(ring);
3903                         amdgpu_bo_kunmap(ring->mqd_obj);
3904                         ring->mqd_ptr = NULL;
3905                 }
3906                 amdgpu_bo_unreserve(ring->mqd_obj);
3907                 if (r)
3908                         goto done;
3909         }
3910
3911         r = amdgpu_gfx_enable_kcq(adev);
3912 done:
3913         return r;
3914 }
3915
3916 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3917 {
3918         int r, i;
3919         struct amdgpu_ring *ring;
3920
3921         if (!(adev->flags & AMD_IS_APU))
3922                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3923
3924         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3925                 if (adev->gfx.num_gfx_rings) {
3926                         /* legacy firmware loading */
3927                         r = gfx_v9_0_cp_gfx_load_microcode(adev);
3928                         if (r)
3929                                 return r;
3930                 }
3931
3932                 r = gfx_v9_0_cp_compute_load_microcode(adev);
3933                 if (r)
3934                         return r;
3935         }
3936
3937         r = gfx_v9_0_kiq_resume(adev);
3938         if (r)
3939                 return r;
3940
3941         if (adev->gfx.num_gfx_rings) {
3942                 r = gfx_v9_0_cp_gfx_resume(adev);
3943                 if (r)
3944                         return r;
3945         }
3946
3947         r = gfx_v9_0_kcq_resume(adev);
3948         if (r)
3949                 return r;
3950
3951         if (adev->gfx.num_gfx_rings) {
3952                 ring = &adev->gfx.gfx_ring[0];
3953                 r = amdgpu_ring_test_helper(ring);
3954                 if (r)
3955                         return r;
3956         }
3957
3958         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3959                 ring = &adev->gfx.compute_ring[i];
3960                 amdgpu_ring_test_helper(ring);
3961         }
3962
3963         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3964
3965         return 0;
3966 }
3967
3968 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3969 {
3970         u32 tmp;
3971
3972         if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1) &&
3973             adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 2))
3974                 return;
3975
3976         tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3977         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3978                                 adev->df.hash_status.hash_64k);
3979         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3980                                 adev->df.hash_status.hash_2m);
3981         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3982                                 adev->df.hash_status.hash_1g);
3983         WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3984 }
3985
3986 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3987 {
3988         if (adev->gfx.num_gfx_rings)
3989                 gfx_v9_0_cp_gfx_enable(adev, enable);
3990         gfx_v9_0_cp_compute_enable(adev, enable);
3991 }
3992
3993 static int gfx_v9_0_hw_init(void *handle)
3994 {
3995         int r;
3996         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3997
3998         if (!amdgpu_sriov_vf(adev))
3999                 gfx_v9_0_init_golden_registers(adev);
4000
4001         gfx_v9_0_constants_init(adev);
4002
4003         gfx_v9_0_init_tcp_config(adev);
4004
4005         r = adev->gfx.rlc.funcs->resume(adev);
4006         if (r)
4007                 return r;
4008
4009         r = gfx_v9_0_cp_resume(adev);
4010         if (r)
4011                 return r;
4012
4013         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
4014                 gfx_v9_4_2_set_power_brake_sequence(adev);
4015
4016         return r;
4017 }
4018
4019 static int gfx_v9_0_hw_fini(void *handle)
4020 {
4021         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4022
4023         amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4024         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4025         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4026
4027         /* DF freeze and kcq disable will fail */
4028         if (!amdgpu_ras_intr_triggered())
4029                 /* disable KCQ to avoid CPC touch memory not valid anymore */
4030                 amdgpu_gfx_disable_kcq(adev);
4031
4032         if (amdgpu_sriov_vf(adev)) {
4033                 gfx_v9_0_cp_gfx_enable(adev, false);
4034                 /* must disable polling for SRIOV when hw finished, otherwise
4035                  * CPC engine may still keep fetching WB address which is already
4036                  * invalid after sw finished and trigger DMAR reading error in
4037                  * hypervisor side.
4038                  */
4039                 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4040                 return 0;
4041         }
4042
4043         /* Use deinitialize sequence from CAIL when unbinding device from driver,
4044          * otherwise KIQ is hanging when binding back
4045          */
4046         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4047                 mutex_lock(&adev->srbm_mutex);
4048                 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
4049                                 adev->gfx.kiq.ring.pipe,
4050                                 adev->gfx.kiq.ring.queue, 0);
4051                 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
4052                 soc15_grbm_select(adev, 0, 0, 0, 0);
4053                 mutex_unlock(&adev->srbm_mutex);
4054         }
4055
4056         gfx_v9_0_cp_enable(adev, false);
4057
4058         /* Skip suspend with A+A reset */
4059         if (adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) {
4060                 dev_dbg(adev->dev, "Device in reset. Skipping RLC halt\n");
4061                 return 0;
4062         }
4063
4064         adev->gfx.rlc.funcs->stop(adev);
4065         return 0;
4066 }
4067
4068 static int gfx_v9_0_suspend(void *handle)
4069 {
4070         return gfx_v9_0_hw_fini(handle);
4071 }
4072
4073 static int gfx_v9_0_resume(void *handle)
4074 {
4075         return gfx_v9_0_hw_init(handle);
4076 }
4077
4078 static bool gfx_v9_0_is_idle(void *handle)
4079 {
4080         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4081
4082         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4083                                 GRBM_STATUS, GUI_ACTIVE))
4084                 return false;
4085         else
4086                 return true;
4087 }
4088
4089 static int gfx_v9_0_wait_for_idle(void *handle)
4090 {
4091         unsigned i;
4092         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4093
4094         for (i = 0; i < adev->usec_timeout; i++) {
4095                 if (gfx_v9_0_is_idle(handle))
4096                         return 0;
4097                 udelay(1);
4098         }
4099         return -ETIMEDOUT;
4100 }
4101
4102 static int gfx_v9_0_soft_reset(void *handle)
4103 {
4104         u32 grbm_soft_reset = 0;
4105         u32 tmp;
4106         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4107
4108         /* GRBM_STATUS */
4109         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4110         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4111                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4112                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4113                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4114                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4115                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4116                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4117                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4118                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4119                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4120         }
4121
4122         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4123                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4124                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4125         }
4126
4127         /* GRBM_STATUS2 */
4128         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4129         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4130                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4131                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4132
4133
4134         if (grbm_soft_reset) {
4135                 /* stop the rlc */
4136                 adev->gfx.rlc.funcs->stop(adev);
4137
4138                 if (adev->gfx.num_gfx_rings)
4139                         /* Disable GFX parsing/prefetching */
4140                         gfx_v9_0_cp_gfx_enable(adev, false);
4141
4142                 /* Disable MEC parsing/prefetching */
4143                 gfx_v9_0_cp_compute_enable(adev, false);
4144
4145                 if (grbm_soft_reset) {
4146                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4147                         tmp |= grbm_soft_reset;
4148                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4149                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4150                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4151
4152                         udelay(50);
4153
4154                         tmp &= ~grbm_soft_reset;
4155                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4156                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4157                 }
4158
4159                 /* Wait a little for things to settle down */
4160                 udelay(50);
4161         }
4162         return 0;
4163 }
4164
4165 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4166 {
4167         signed long r, cnt = 0;
4168         unsigned long flags;
4169         uint32_t seq, reg_val_offs = 0;
4170         uint64_t value = 0;
4171         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4172         struct amdgpu_ring *ring = &kiq->ring;
4173
4174         BUG_ON(!ring->funcs->emit_rreg);
4175
4176         spin_lock_irqsave(&kiq->ring_lock, flags);
4177         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
4178                 pr_err("critical bug! too many kiq readers\n");
4179                 goto failed_unlock;
4180         }
4181         amdgpu_ring_alloc(ring, 32);
4182         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4183         amdgpu_ring_write(ring, 9 |     /* src: register*/
4184                                 (5 << 8) |      /* dst: memory */
4185                                 (1 << 16) |     /* count sel */
4186                                 (1 << 20));     /* write confirm */
4187         amdgpu_ring_write(ring, 0);
4188         amdgpu_ring_write(ring, 0);
4189         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4190                                 reg_val_offs * 4));
4191         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4192                                 reg_val_offs * 4));
4193         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4194         if (r)
4195                 goto failed_undo;
4196
4197         amdgpu_ring_commit(ring);
4198         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4199
4200         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4201
4202         /* don't wait anymore for gpu reset case because this way may
4203          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4204          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4205          * never return if we keep waiting in virt_kiq_rreg, which cause
4206          * gpu_recover() hang there.
4207          *
4208          * also don't wait anymore for IRQ context
4209          * */
4210         if (r < 1 && (amdgpu_in_reset(adev)))
4211                 goto failed_kiq_read;
4212
4213         might_sleep();
4214         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4215                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4216                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4217         }
4218
4219         if (cnt > MAX_KIQ_REG_TRY)
4220                 goto failed_kiq_read;
4221
4222         mb();
4223         value = (uint64_t)adev->wb.wb[reg_val_offs] |
4224                 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4225         amdgpu_device_wb_free(adev, reg_val_offs);
4226         return value;
4227
4228 failed_undo:
4229         amdgpu_ring_undo(ring);
4230 failed_unlock:
4231         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4232 failed_kiq_read:
4233         if (reg_val_offs)
4234                 amdgpu_device_wb_free(adev, reg_val_offs);
4235         pr_err("failed to read gpu clock\n");
4236         return ~0;
4237 }
4238
4239 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4240 {
4241         uint64_t clock;
4242
4243         amdgpu_gfx_off_ctrl(adev, false);
4244         mutex_lock(&adev->gfx.gpu_clock_mutex);
4245         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 0, 1) && amdgpu_sriov_runtime(adev)) {
4246                 clock = gfx_v9_0_kiq_read_clock(adev);
4247         } else {
4248                 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4249                 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4250                         ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4251         }
4252         mutex_unlock(&adev->gfx.gpu_clock_mutex);
4253         amdgpu_gfx_off_ctrl(adev, true);
4254         return clock;
4255 }
4256
4257 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4258                                           uint32_t vmid,
4259                                           uint32_t gds_base, uint32_t gds_size,
4260                                           uint32_t gws_base, uint32_t gws_size,
4261                                           uint32_t oa_base, uint32_t oa_size)
4262 {
4263         struct amdgpu_device *adev = ring->adev;
4264
4265         /* GDS Base */
4266         gfx_v9_0_write_data_to_reg(ring, 0, false,
4267                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4268                                    gds_base);
4269
4270         /* GDS Size */
4271         gfx_v9_0_write_data_to_reg(ring, 0, false,
4272                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4273                                    gds_size);
4274
4275         /* GWS */
4276         gfx_v9_0_write_data_to_reg(ring, 0, false,
4277                                    SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4278                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4279
4280         /* OA */
4281         gfx_v9_0_write_data_to_reg(ring, 0, false,
4282                                    SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4283                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
4284 }
4285
4286 static const u32 vgpr_init_compute_shader[] =
4287 {
4288         0xb07c0000, 0xbe8000ff,
4289         0x000000f8, 0xbf110800,
4290         0x7e000280, 0x7e020280,
4291         0x7e040280, 0x7e060280,
4292         0x7e080280, 0x7e0a0280,
4293         0x7e0c0280, 0x7e0e0280,
4294         0x80808800, 0xbe803200,
4295         0xbf84fff5, 0xbf9c0000,
4296         0xd28c0001, 0x0001007f,
4297         0xd28d0001, 0x0002027e,
4298         0x10020288, 0xb8810904,
4299         0xb7814000, 0xd1196a01,
4300         0x00000301, 0xbe800087,
4301         0xbefc00c1, 0xd89c4000,
4302         0x00020201, 0xd89cc080,
4303         0x00040401, 0x320202ff,
4304         0x00000800, 0x80808100,
4305         0xbf84fff8, 0x7e020280,
4306         0xbf810000, 0x00000000,
4307 };
4308
4309 static const u32 sgpr_init_compute_shader[] =
4310 {
4311         0xb07c0000, 0xbe8000ff,
4312         0x0000005f, 0xbee50080,
4313         0xbe812c65, 0xbe822c65,
4314         0xbe832c65, 0xbe842c65,
4315         0xbe852c65, 0xb77c0005,
4316         0x80808500, 0xbf84fff8,
4317         0xbe800080, 0xbf810000,
4318 };
4319
4320 static const u32 vgpr_init_compute_shader_arcturus[] = {
4321         0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4322         0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4323         0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4324         0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4325         0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4326         0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4327         0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4328         0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4329         0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4330         0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4331         0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4332         0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4333         0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4334         0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4335         0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4336         0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4337         0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4338         0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4339         0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4340         0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4341         0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4342         0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4343         0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4344         0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4345         0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4346         0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4347         0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4348         0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4349         0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4350         0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4351         0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4352         0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4353         0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4354         0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4355         0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4356         0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4357         0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4358         0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4359         0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4360         0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4361         0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4362         0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4363         0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4364         0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4365         0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4366         0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4367         0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4368         0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4369         0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4370         0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4371         0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4372         0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4373         0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4374         0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4375         0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4376         0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4377         0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4378         0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4379         0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4380         0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4381         0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4382         0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4383         0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4384         0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4385         0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4386         0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4387         0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4388         0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4389         0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4390         0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4391         0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4392         0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4393         0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4394         0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4395         0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4396         0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4397         0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4398         0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4399         0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4400         0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4401         0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4402         0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4403         0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4404         0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4405         0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4406         0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4407         0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4408         0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4409         0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4410         0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4411         0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4412         0xbf84fff8, 0xbf810000,
4413 };
4414
4415 /* When below register arrays changed, please update gpr_reg_size,
4416   and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4417   to cover all gfx9 ASICs */
4418 static const struct soc15_reg_entry vgpr_init_regs[] = {
4419    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4420    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4421    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4422    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4423    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4424    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4425    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4426    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4427    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4428    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4429    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4430    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4431    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4432    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4433 };
4434
4435 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4436    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4437    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4438    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4439    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4440    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4441    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4442    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4443    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4444    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4445    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4446    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4447    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4448    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4449    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4450 };
4451
4452 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4453    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4454    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4455    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4456    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4457    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4458    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4459    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4460    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4461    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4462    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4463    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4464    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4465    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4466    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4467 };
4468
4469 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4470    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4471    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4472    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4473    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4474    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4475    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4476    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4477    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4478    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4479    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4480    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4481    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4482    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4483    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4484 };
4485
4486 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4487    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4488    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4489    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4490    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4491    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4492    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4493    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4494    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4495    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4496    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4497    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4498    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4499    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4500    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4501    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4502    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4503    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4504    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4505    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4506    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4507    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4508    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4509    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4510    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4511    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4512    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4513    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4514    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4515    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4516    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4517    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4518    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4519    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4520 };
4521
4522 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4523 {
4524         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4525         int i, r;
4526
4527         /* only support when RAS is enabled */
4528         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4529                 return 0;
4530
4531         r = amdgpu_ring_alloc(ring, 7);
4532         if (r) {
4533                 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4534                         ring->name, r);
4535                 return r;
4536         }
4537
4538         WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4539         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4540
4541         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4542         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4543                                 PACKET3_DMA_DATA_DST_SEL(1) |
4544                                 PACKET3_DMA_DATA_SRC_SEL(2) |
4545                                 PACKET3_DMA_DATA_ENGINE(0)));
4546         amdgpu_ring_write(ring, 0);
4547         amdgpu_ring_write(ring, 0);
4548         amdgpu_ring_write(ring, 0);
4549         amdgpu_ring_write(ring, 0);
4550         amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4551                                 adev->gds.gds_size);
4552
4553         amdgpu_ring_commit(ring);
4554
4555         for (i = 0; i < adev->usec_timeout; i++) {
4556                 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4557                         break;
4558                 udelay(1);
4559         }
4560
4561         if (i >= adev->usec_timeout)
4562                 r = -ETIMEDOUT;
4563
4564         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4565
4566         return r;
4567 }
4568
4569 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4570 {
4571         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4572         struct amdgpu_ib ib;
4573         struct dma_fence *f = NULL;
4574         int r, i;
4575         unsigned total_size, vgpr_offset, sgpr_offset;
4576         u64 gpu_addr;
4577
4578         int compute_dim_x = adev->gfx.config.max_shader_engines *
4579                                                 adev->gfx.config.max_cu_per_sh *
4580                                                 adev->gfx.config.max_sh_per_se;
4581         int sgpr_work_group_size = 5;
4582         int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4583         int vgpr_init_shader_size;
4584         const u32 *vgpr_init_shader_ptr;
4585         const struct soc15_reg_entry *vgpr_init_regs_ptr;
4586
4587         /* only support when RAS is enabled */
4588         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4589                 return 0;
4590
4591         /* bail if the compute ring is not ready */
4592         if (!ring->sched.ready)
4593                 return 0;
4594
4595         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) {
4596                 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4597                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4598                 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4599         } else {
4600                 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4601                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4602                 vgpr_init_regs_ptr = vgpr_init_regs;
4603         }
4604
4605         total_size =
4606                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4607         total_size +=
4608                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4609         total_size +=
4610                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4611         total_size = ALIGN(total_size, 256);
4612         vgpr_offset = total_size;
4613         total_size += ALIGN(vgpr_init_shader_size, 256);
4614         sgpr_offset = total_size;
4615         total_size += sizeof(sgpr_init_compute_shader);
4616
4617         /* allocate an indirect buffer to put the commands in */
4618         memset(&ib, 0, sizeof(ib));
4619         r = amdgpu_ib_get(adev, NULL, total_size,
4620                                         AMDGPU_IB_POOL_DIRECT, &ib);
4621         if (r) {
4622                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4623                 return r;
4624         }
4625
4626         /* load the compute shaders */
4627         for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4628                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4629
4630         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4631                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4632
4633         /* init the ib length to 0 */
4634         ib.length_dw = 0;
4635
4636         /* VGPR */
4637         /* write the register state for the compute dispatch */
4638         for (i = 0; i < gpr_reg_size; i++) {
4639                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4640                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4641                                                                 - PACKET3_SET_SH_REG_START;
4642                 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4643         }
4644         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4645         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4646         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4647         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4648                                                         - PACKET3_SET_SH_REG_START;
4649         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4650         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4651
4652         /* write dispatch packet */
4653         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4654         ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4655         ib.ptr[ib.length_dw++] = 1; /* y */
4656         ib.ptr[ib.length_dw++] = 1; /* z */
4657         ib.ptr[ib.length_dw++] =
4658                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4659
4660         /* write CS partial flush packet */
4661         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4662         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4663
4664         /* SGPR1 */
4665         /* write the register state for the compute dispatch */
4666         for (i = 0; i < gpr_reg_size; i++) {
4667                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4668                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4669                                                                 - PACKET3_SET_SH_REG_START;
4670                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4671         }
4672         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4673         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4674         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4675         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4676                                                         - PACKET3_SET_SH_REG_START;
4677         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4678         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4679
4680         /* write dispatch packet */
4681         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4682         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4683         ib.ptr[ib.length_dw++] = 1; /* y */
4684         ib.ptr[ib.length_dw++] = 1; /* z */
4685         ib.ptr[ib.length_dw++] =
4686                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4687
4688         /* write CS partial flush packet */
4689         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4690         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4691
4692         /* SGPR2 */
4693         /* write the register state for the compute dispatch */
4694         for (i = 0; i < gpr_reg_size; i++) {
4695                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4696                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4697                                                                 - PACKET3_SET_SH_REG_START;
4698                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4699         }
4700         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4701         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4702         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4703         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4704                                                         - PACKET3_SET_SH_REG_START;
4705         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4706         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4707
4708         /* write dispatch packet */
4709         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4710         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4711         ib.ptr[ib.length_dw++] = 1; /* y */
4712         ib.ptr[ib.length_dw++] = 1; /* z */
4713         ib.ptr[ib.length_dw++] =
4714                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4715
4716         /* write CS partial flush packet */
4717         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4718         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4719
4720         /* shedule the ib on the ring */
4721         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4722         if (r) {
4723                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4724                 goto fail;
4725         }
4726
4727         /* wait for the GPU to finish processing the IB */
4728         r = dma_fence_wait(f, false);
4729         if (r) {
4730                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4731                 goto fail;
4732         }
4733
4734 fail:
4735         amdgpu_ib_free(adev, &ib, NULL);
4736         dma_fence_put(f);
4737
4738         return r;
4739 }
4740
4741 static int gfx_v9_0_early_init(void *handle)
4742 {
4743         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4744
4745         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) ||
4746             adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
4747                 adev->gfx.num_gfx_rings = 0;
4748         else
4749                 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4750         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4751                                           AMDGPU_MAX_COMPUTE_RINGS);
4752         gfx_v9_0_set_kiq_pm4_funcs(adev);
4753         gfx_v9_0_set_ring_funcs(adev);
4754         gfx_v9_0_set_irq_funcs(adev);
4755         gfx_v9_0_set_gds_init(adev);
4756         gfx_v9_0_set_rlc_funcs(adev);
4757
4758         return 0;
4759 }
4760
4761 static int gfx_v9_0_ecc_late_init(void *handle)
4762 {
4763         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4764         int r;
4765
4766         /*
4767          * Temp workaround to fix the issue that CP firmware fails to
4768          * update read pointer when CPDMA is writing clearing operation
4769          * to GDS in suspend/resume sequence on several cards. So just
4770          * limit this operation in cold boot sequence.
4771          */
4772         if ((!adev->in_suspend) &&
4773             (adev->gds.gds_size)) {
4774                 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4775                 if (r)
4776                         return r;
4777         }
4778
4779         /* requires IBs so do in late init after IB pool is initialized */
4780         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
4781                 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4782         else
4783                 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4784
4785         if (r)
4786                 return r;
4787
4788         if (adev->gfx.ras_funcs &&
4789             adev->gfx.ras_funcs->ras_late_init) {
4790                 r = adev->gfx.ras_funcs->ras_late_init(adev);
4791                 if (r)
4792                         return r;
4793         }
4794
4795         if (adev->gfx.ras_funcs &&
4796             adev->gfx.ras_funcs->enable_watchdog_timer)
4797                 adev->gfx.ras_funcs->enable_watchdog_timer(adev);
4798
4799         return 0;
4800 }
4801
4802 static int gfx_v9_0_late_init(void *handle)
4803 {
4804         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4805         int r;
4806
4807         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4808         if (r)
4809                 return r;
4810
4811         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4812         if (r)
4813                 return r;
4814
4815         r = gfx_v9_0_ecc_late_init(handle);
4816         if (r)
4817                 return r;
4818
4819         return 0;
4820 }
4821
4822 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4823 {
4824         uint32_t rlc_setting;
4825
4826         /* if RLC is not enabled, do nothing */
4827         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4828         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4829                 return false;
4830
4831         return true;
4832 }
4833
4834 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4835 {
4836         uint32_t data;
4837         unsigned i;
4838
4839         data = RLC_SAFE_MODE__CMD_MASK;
4840         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4841         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4842
4843         /* wait for RLC_SAFE_MODE */
4844         for (i = 0; i < adev->usec_timeout; i++) {
4845                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4846                         break;
4847                 udelay(1);
4848         }
4849 }
4850
4851 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4852 {
4853         uint32_t data;
4854
4855         data = RLC_SAFE_MODE__CMD_MASK;
4856         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4857 }
4858
4859 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4860                                                 bool enable)
4861 {
4862         amdgpu_gfx_rlc_enter_safe_mode(adev);
4863
4864         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4865                 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4866                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4867                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4868         } else {
4869                 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4870                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4871                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4872         }
4873
4874         amdgpu_gfx_rlc_exit_safe_mode(adev);
4875 }
4876
4877 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4878                                                 bool enable)
4879 {
4880         /* TODO: double check if we need to perform under safe mode */
4881         /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4882
4883         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4884                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4885         else
4886                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4887
4888         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4889                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4890         else
4891                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4892
4893         /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4894 }
4895
4896 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4897                                                       bool enable)
4898 {
4899         uint32_t data, def;
4900
4901         amdgpu_gfx_rlc_enter_safe_mode(adev);
4902
4903         /* It is disabled by HW by default */
4904         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4905                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4906                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4907
4908                 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 2, 1))
4909                         data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4910
4911                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4912                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4913                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4914
4915                 /* only for Vega10 & Raven1 */
4916                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4917
4918                 if (def != data)
4919                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4920
4921                 /* MGLS is a global flag to control all MGLS in GFX */
4922                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4923                         /* 2 - RLC memory Light sleep */
4924                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4925                                 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4926                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4927                                 if (def != data)
4928                                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4929                         }
4930                         /* 3 - CP memory Light sleep */
4931                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4932                                 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4933                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4934                                 if (def != data)
4935                                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4936                         }
4937                 }
4938         } else {
4939                 /* 1 - MGCG_OVERRIDE */
4940                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4941
4942                 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 2, 1))
4943                         data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4944
4945                 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4946                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4947                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4948                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4949
4950                 if (def != data)
4951                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4952
4953                 /* 2 - disable MGLS in RLC */
4954                 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4955                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4956                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4957                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4958                 }
4959
4960                 /* 3 - disable MGLS in CP */
4961                 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4962                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4963                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4964                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4965                 }
4966         }
4967
4968         amdgpu_gfx_rlc_exit_safe_mode(adev);
4969 }
4970
4971 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4972                                            bool enable)
4973 {
4974         uint32_t data, def;
4975
4976         if (!adev->gfx.num_gfx_rings)
4977                 return;
4978
4979         amdgpu_gfx_rlc_enter_safe_mode(adev);
4980
4981         /* Enable 3D CGCG/CGLS */
4982         if (enable) {
4983                 /* write cmd to clear cgcg/cgls ov */
4984                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4985                 /* unset CGCG override */
4986                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4987                 /* update CGCG and CGLS override bits */
4988                 if (def != data)
4989                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4990
4991                 /* enable 3Dcgcg FSM(0x0000363f) */
4992                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4993
4994                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
4995                         data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4996                                 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4997                 else
4998                         data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
4999
5000                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5001                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5002                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5003                 if (def != data)
5004                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5005
5006                 /* set IDLE_POLL_COUNT(0x00900100) */
5007                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5008                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5009                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5010                 if (def != data)
5011                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5012         } else {
5013                 /* Disable CGCG/CGLS */
5014                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5015                 /* disable cgcg, cgls should be disabled */
5016                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
5017                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
5018                 /* disable cgcg and cgls in FSM */
5019                 if (def != data)
5020                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5021         }
5022
5023         amdgpu_gfx_rlc_exit_safe_mode(adev);
5024 }
5025
5026 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5027                                                       bool enable)
5028 {
5029         uint32_t def, data;
5030
5031         amdgpu_gfx_rlc_enter_safe_mode(adev);
5032
5033         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5034                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5035                 /* unset CGCG override */
5036                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5037                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5038                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5039                 else
5040                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5041                 /* update CGCG and CGLS override bits */
5042                 if (def != data)
5043                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5044
5045                 /* enable cgcg FSM(0x0000363F) */
5046                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5047
5048                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1))
5049                         data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5050                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5051                 else
5052                         data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5053                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5054                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5055                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5056                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5057                 if (def != data)
5058                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5059
5060                 /* set IDLE_POLL_COUNT(0x00900100) */
5061                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5062                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5063                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5064                 if (def != data)
5065                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5066         } else {
5067                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5068                 /* reset CGCG/CGLS bits */
5069                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5070                 /* disable cgcg and cgls in FSM */
5071                 if (def != data)
5072                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5073         }
5074
5075         amdgpu_gfx_rlc_exit_safe_mode(adev);
5076 }
5077
5078 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5079                                             bool enable)
5080 {
5081         if (enable) {
5082                 /* CGCG/CGLS should be enabled after MGCG/MGLS
5083                  * ===  MGCG + MGLS ===
5084                  */
5085                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5086                 /* ===  CGCG /CGLS for GFX 3D Only === */
5087                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5088                 /* ===  CGCG + CGLS === */
5089                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5090         } else {
5091                 /* CGCG/CGLS should be disabled before MGCG/MGLS
5092                  * ===  CGCG + CGLS ===
5093                  */
5094                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5095                 /* ===  CGCG /CGLS for GFX 3D Only === */
5096                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5097                 /* ===  MGCG + MGLS === */
5098                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5099         }
5100         return 0;
5101 }
5102
5103 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
5104 {
5105         u32 reg, data;
5106
5107         amdgpu_gfx_off_ctrl(adev, false);
5108
5109         reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5110         if (amdgpu_sriov_is_pp_one_vf(adev))
5111                 data = RREG32_NO_KIQ(reg);
5112         else
5113                 data = RREG32(reg);
5114
5115         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5116         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5117
5118         if (amdgpu_sriov_is_pp_one_vf(adev))
5119                 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5120         else
5121                 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5122
5123         amdgpu_gfx_off_ctrl(adev, true);
5124 }
5125
5126 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5127                                         uint32_t offset,
5128                                         struct soc15_reg_rlcg *entries, int arr_size)
5129 {
5130         int i;
5131         uint32_t reg;
5132
5133         if (!entries)
5134                 return false;
5135
5136         for (i = 0; i < arr_size; i++) {
5137                 const struct soc15_reg_rlcg *entry;
5138
5139                 entry = &entries[i];
5140                 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5141                 if (offset == reg)
5142                         return true;
5143         }
5144
5145         return false;
5146 }
5147
5148 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5149 {
5150         return gfx_v9_0_check_rlcg_range(adev, offset,
5151                                         (void *)rlcg_access_gc_9_0,
5152                                         ARRAY_SIZE(rlcg_access_gc_9_0));
5153 }
5154
5155 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5156         .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5157         .set_safe_mode = gfx_v9_0_set_safe_mode,
5158         .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5159         .init = gfx_v9_0_rlc_init,
5160         .get_csb_size = gfx_v9_0_get_csb_size,
5161         .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5162         .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5163         .resume = gfx_v9_0_rlc_resume,
5164         .stop = gfx_v9_0_rlc_stop,
5165         .reset = gfx_v9_0_rlc_reset,
5166         .start = gfx_v9_0_rlc_start,
5167         .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5168         .sriov_wreg = gfx_v9_0_sriov_wreg,
5169         .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5170 };
5171
5172 static int gfx_v9_0_set_powergating_state(void *handle,
5173                                           enum amd_powergating_state state)
5174 {
5175         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5176         bool enable = (state == AMD_PG_STATE_GATE);
5177
5178         switch (adev->ip_versions[GC_HWIP][0]) {
5179         case IP_VERSION(9, 2, 2):
5180         case IP_VERSION(9, 1, 0):
5181         case IP_VERSION(9, 3, 0):
5182                 if (!enable)
5183                         amdgpu_gfx_off_ctrl(adev, false);
5184
5185                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5186                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5187                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5188                 } else {
5189                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5190                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5191                 }
5192
5193                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5194                         gfx_v9_0_enable_cp_power_gating(adev, true);
5195                 else
5196                         gfx_v9_0_enable_cp_power_gating(adev, false);
5197
5198                 /* update gfx cgpg state */
5199                 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5200
5201                 /* update mgcg state */
5202                 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5203
5204                 if (enable)
5205                         amdgpu_gfx_off_ctrl(adev, true);
5206                 break;
5207         case IP_VERSION(9, 2, 1):
5208                 amdgpu_gfx_off_ctrl(adev, enable);
5209                 break;
5210         default:
5211                 break;
5212         }
5213
5214         return 0;
5215 }
5216
5217 static int gfx_v9_0_set_clockgating_state(void *handle,
5218                                           enum amd_clockgating_state state)
5219 {
5220         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5221
5222         if (amdgpu_sriov_vf(adev))
5223                 return 0;
5224
5225         switch (adev->ip_versions[GC_HWIP][0]) {
5226         case IP_VERSION(9, 0, 1):
5227         case IP_VERSION(9, 2, 1):
5228         case IP_VERSION(9, 4, 0):
5229         case IP_VERSION(9, 2, 2):
5230         case IP_VERSION(9, 1, 0):
5231         case IP_VERSION(9, 4, 1):
5232         case IP_VERSION(9, 3, 0):
5233         case IP_VERSION(9, 4, 2):
5234                 gfx_v9_0_update_gfx_clock_gating(adev,
5235                                                  state == AMD_CG_STATE_GATE);
5236                 break;
5237         default:
5238                 break;
5239         }
5240         return 0;
5241 }
5242
5243 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
5244 {
5245         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5246         int data;
5247
5248         if (amdgpu_sriov_vf(adev))
5249                 *flags = 0;
5250
5251         /* AMD_CG_SUPPORT_GFX_MGCG */
5252         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5253         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5254                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5255
5256         /* AMD_CG_SUPPORT_GFX_CGCG */
5257         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5258         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5259                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5260
5261         /* AMD_CG_SUPPORT_GFX_CGLS */
5262         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5263                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5264
5265         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5266         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5267         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5268                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5269
5270         /* AMD_CG_SUPPORT_GFX_CP_LS */
5271         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5272         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5273                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5274
5275         if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) {
5276                 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5277                 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5278                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5279                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5280
5281                 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5282                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5283                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5284         }
5285 }
5286
5287 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5288 {
5289         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
5290 }
5291
5292 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5293 {
5294         struct amdgpu_device *adev = ring->adev;
5295         u64 wptr;
5296
5297         /* XXX check if swapping is necessary on BE */
5298         if (ring->use_doorbell) {
5299                 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
5300         } else {
5301                 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5302                 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5303         }
5304
5305         return wptr;
5306 }
5307
5308 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5309 {
5310         struct amdgpu_device *adev = ring->adev;
5311
5312         if (ring->use_doorbell) {
5313                 /* XXX check if swapping is necessary on BE */
5314                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5315                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5316         } else {
5317                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5318                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5319         }
5320 }
5321
5322 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5323 {
5324         struct amdgpu_device *adev = ring->adev;
5325         u32 ref_and_mask, reg_mem_engine;
5326         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5327
5328         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5329                 switch (ring->me) {
5330                 case 1:
5331                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5332                         break;
5333                 case 2:
5334                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5335                         break;
5336                 default:
5337                         return;
5338                 }
5339                 reg_mem_engine = 0;
5340         } else {
5341                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5342                 reg_mem_engine = 1; /* pfp */
5343         }
5344
5345         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5346                               adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5347                               adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5348                               ref_and_mask, ref_and_mask, 0x20);
5349 }
5350
5351 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5352                                         struct amdgpu_job *job,
5353                                         struct amdgpu_ib *ib,
5354                                         uint32_t flags)
5355 {
5356         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5357         u32 header, control = 0;
5358
5359         if (ib->flags & AMDGPU_IB_FLAG_CE)
5360                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5361         else
5362                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5363
5364         control |= ib->length_dw | (vmid << 24);
5365
5366         if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5367                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5368
5369                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5370                         gfx_v9_0_ring_emit_de_meta(ring);
5371         }
5372
5373         amdgpu_ring_write(ring, header);
5374         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5375         amdgpu_ring_write(ring,
5376 #ifdef __BIG_ENDIAN
5377                 (2 << 0) |
5378 #endif
5379                 lower_32_bits(ib->gpu_addr));
5380         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5381         amdgpu_ring_write(ring, control);
5382 }
5383
5384 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5385                                           struct amdgpu_job *job,
5386                                           struct amdgpu_ib *ib,
5387                                           uint32_t flags)
5388 {
5389         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5390         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5391
5392         /* Currently, there is a high possibility to get wave ID mismatch
5393          * between ME and GDS, leading to a hw deadlock, because ME generates
5394          * different wave IDs than the GDS expects. This situation happens
5395          * randomly when at least 5 compute pipes use GDS ordered append.
5396          * The wave IDs generated by ME are also wrong after suspend/resume.
5397          * Those are probably bugs somewhere else in the kernel driver.
5398          *
5399          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5400          * GDS to 0 for this ring (me/pipe).
5401          */
5402         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5403                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5404                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5405                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5406         }
5407
5408         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5409         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5410         amdgpu_ring_write(ring,
5411 #ifdef __BIG_ENDIAN
5412                                 (2 << 0) |
5413 #endif
5414                                 lower_32_bits(ib->gpu_addr));
5415         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5416         amdgpu_ring_write(ring, control);
5417 }
5418
5419 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5420                                      u64 seq, unsigned flags)
5421 {
5422         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5423         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5424         bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5425
5426         /* RELEASE_MEM - flush caches, send int */
5427         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5428         amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
5429                                                EOP_TC_NC_ACTION_EN) :
5430                                               (EOP_TCL1_ACTION_EN |
5431                                                EOP_TC_ACTION_EN |
5432                                                EOP_TC_WB_ACTION_EN |
5433                                                EOP_TC_MD_ACTION_EN)) |
5434                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5435                                  EVENT_INDEX(5)));
5436         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5437
5438         /*
5439          * the address should be Qword aligned if 64bit write, Dword
5440          * aligned if only send 32bit data low (discard data high)
5441          */
5442         if (write64bit)
5443                 BUG_ON(addr & 0x7);
5444         else
5445                 BUG_ON(addr & 0x3);
5446         amdgpu_ring_write(ring, lower_32_bits(addr));
5447         amdgpu_ring_write(ring, upper_32_bits(addr));
5448         amdgpu_ring_write(ring, lower_32_bits(seq));
5449         amdgpu_ring_write(ring, upper_32_bits(seq));
5450         amdgpu_ring_write(ring, 0);
5451 }
5452
5453 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5454 {
5455         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5456         uint32_t seq = ring->fence_drv.sync_seq;
5457         uint64_t addr = ring->fence_drv.gpu_addr;
5458
5459         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5460                               lower_32_bits(addr), upper_32_bits(addr),
5461                               seq, 0xffffffff, 4);
5462 }
5463
5464 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5465                                         unsigned vmid, uint64_t pd_addr)
5466 {
5467         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5468
5469         /* compute doesn't have PFP */
5470         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5471                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5472                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5473                 amdgpu_ring_write(ring, 0x0);
5474         }
5475 }
5476
5477 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5478 {
5479         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
5480 }
5481
5482 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5483 {
5484         u64 wptr;
5485
5486         /* XXX check if swapping is necessary on BE */
5487         if (ring->use_doorbell)
5488                 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
5489         else
5490                 BUG();
5491         return wptr;
5492 }
5493
5494 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5495 {
5496         struct amdgpu_device *adev = ring->adev;
5497
5498         /* XXX check if swapping is necessary on BE */
5499         if (ring->use_doorbell) {
5500                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5501                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5502         } else{
5503                 BUG(); /* only DOORBELL method supported on gfx9 now */
5504         }
5505 }
5506
5507 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5508                                          u64 seq, unsigned int flags)
5509 {
5510         struct amdgpu_device *adev = ring->adev;
5511
5512         /* we only allocate 32bit for each seq wb address */
5513         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5514
5515         /* write fence seq to the "addr" */
5516         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5517         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5518                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5519         amdgpu_ring_write(ring, lower_32_bits(addr));
5520         amdgpu_ring_write(ring, upper_32_bits(addr));
5521         amdgpu_ring_write(ring, lower_32_bits(seq));
5522
5523         if (flags & AMDGPU_FENCE_FLAG_INT) {
5524                 /* set register to trigger INT */
5525                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5526                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5527                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5528                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5529                 amdgpu_ring_write(ring, 0);
5530                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5531         }
5532 }
5533
5534 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5535 {
5536         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5537         amdgpu_ring_write(ring, 0);
5538 }
5539
5540 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
5541 {
5542         struct v9_ce_ib_state ce_payload = {0};
5543         uint64_t csa_addr;
5544         int cnt;
5545
5546         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5547         csa_addr = amdgpu_csa_vaddr(ring->adev);
5548
5549         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5550         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5551                                  WRITE_DATA_DST_SEL(8) |
5552                                  WR_CONFIRM) |
5553                                  WRITE_DATA_CACHE_POLICY(0));
5554         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5555         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5556         amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
5557 }
5558
5559 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
5560 {
5561         struct v9_de_ib_state de_payload = {0};
5562         uint64_t csa_addr, gds_addr;
5563         int cnt;
5564
5565         csa_addr = amdgpu_csa_vaddr(ring->adev);
5566         gds_addr = csa_addr + 4096;
5567         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5568         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5569
5570         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5571         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5572         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5573                                  WRITE_DATA_DST_SEL(8) |
5574                                  WR_CONFIRM) |
5575                                  WRITE_DATA_CACHE_POLICY(0));
5576         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5577         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5578         amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
5579 }
5580
5581 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5582                                    bool secure)
5583 {
5584         uint32_t v = secure ? FRAME_TMZ : 0;
5585
5586         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5587         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5588 }
5589
5590 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5591 {
5592         uint32_t dw2 = 0;
5593
5594         if (amdgpu_sriov_vf(ring->adev))
5595                 gfx_v9_0_ring_emit_ce_meta(ring);
5596
5597         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5598         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5599                 /* set load_global_config & load_global_uconfig */
5600                 dw2 |= 0x8001;
5601                 /* set load_cs_sh_regs */
5602                 dw2 |= 0x01000000;
5603                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5604                 dw2 |= 0x10002;
5605
5606                 /* set load_ce_ram if preamble presented */
5607                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5608                         dw2 |= 0x10000000;
5609         } else {
5610                 /* still load_ce_ram if this is the first time preamble presented
5611                  * although there is no context switch happens.
5612                  */
5613                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5614                         dw2 |= 0x10000000;
5615         }
5616
5617         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5618         amdgpu_ring_write(ring, dw2);
5619         amdgpu_ring_write(ring, 0);
5620 }
5621
5622 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5623 {
5624         unsigned ret;
5625         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5626         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5627         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5628         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5629         ret = ring->wptr & ring->buf_mask;
5630         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5631         return ret;
5632 }
5633
5634 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5635 {
5636         unsigned cur;
5637         BUG_ON(offset > ring->buf_mask);
5638         BUG_ON(ring->ring[offset] != 0x55aa55aa);
5639
5640         cur = (ring->wptr & ring->buf_mask) - 1;
5641         if (likely(cur > offset))
5642                 ring->ring[offset] = cur - offset;
5643         else
5644                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5645 }
5646
5647 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5648                                     uint32_t reg_val_offs)
5649 {
5650         struct amdgpu_device *adev = ring->adev;
5651
5652         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5653         amdgpu_ring_write(ring, 0 |     /* src: register*/
5654                                 (5 << 8) |      /* dst: memory */
5655                                 (1 << 20));     /* write confirm */
5656         amdgpu_ring_write(ring, reg);
5657         amdgpu_ring_write(ring, 0);
5658         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5659                                 reg_val_offs * 4));
5660         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5661                                 reg_val_offs * 4));
5662 }
5663
5664 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5665                                     uint32_t val)
5666 {
5667         uint32_t cmd = 0;
5668
5669         switch (ring->funcs->type) {
5670         case AMDGPU_RING_TYPE_GFX:
5671                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5672                 break;
5673         case AMDGPU_RING_TYPE_KIQ:
5674                 cmd = (1 << 16); /* no inc addr */
5675                 break;
5676         default:
5677                 cmd = WR_CONFIRM;
5678                 break;
5679         }
5680         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5681         amdgpu_ring_write(ring, cmd);
5682         amdgpu_ring_write(ring, reg);
5683         amdgpu_ring_write(ring, 0);
5684         amdgpu_ring_write(ring, val);
5685 }
5686
5687 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5688                                         uint32_t val, uint32_t mask)
5689 {
5690         gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5691 }
5692
5693 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5694                                                   uint32_t reg0, uint32_t reg1,
5695                                                   uint32_t ref, uint32_t mask)
5696 {
5697         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5698         struct amdgpu_device *adev = ring->adev;
5699         bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5700                 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5701
5702         if (fw_version_ok)
5703                 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5704                                       ref, mask, 0x20);
5705         else
5706                 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5707                                                            ref, mask);
5708 }
5709
5710 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5711 {
5712         struct amdgpu_device *adev = ring->adev;
5713         uint32_t value = 0;
5714
5715         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5716         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5717         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5718         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5719         WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5720 }
5721
5722 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5723                                                  enum amdgpu_interrupt_state state)
5724 {
5725         switch (state) {
5726         case AMDGPU_IRQ_STATE_DISABLE:
5727         case AMDGPU_IRQ_STATE_ENABLE:
5728                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5729                                TIME_STAMP_INT_ENABLE,
5730                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5731                 break;
5732         default:
5733                 break;
5734         }
5735 }
5736
5737 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5738                                                      int me, int pipe,
5739                                                      enum amdgpu_interrupt_state state)
5740 {
5741         u32 mec_int_cntl, mec_int_cntl_reg;
5742
5743         /*
5744          * amdgpu controls only the first MEC. That's why this function only
5745          * handles the setting of interrupts for this specific MEC. All other
5746          * pipes' interrupts are set by amdkfd.
5747          */
5748
5749         if (me == 1) {
5750                 switch (pipe) {
5751                 case 0:
5752                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5753                         break;
5754                 case 1:
5755                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5756                         break;
5757                 case 2:
5758                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5759                         break;
5760                 case 3:
5761                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5762                         break;
5763                 default:
5764                         DRM_DEBUG("invalid pipe %d\n", pipe);
5765                         return;
5766                 }
5767         } else {
5768                 DRM_DEBUG("invalid me %d\n", me);
5769                 return;
5770         }
5771
5772         switch (state) {
5773         case AMDGPU_IRQ_STATE_DISABLE:
5774                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5775                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5776                                              TIME_STAMP_INT_ENABLE, 0);
5777                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5778                 break;
5779         case AMDGPU_IRQ_STATE_ENABLE:
5780                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5781                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5782                                              TIME_STAMP_INT_ENABLE, 1);
5783                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5784                 break;
5785         default:
5786                 break;
5787         }
5788 }
5789
5790 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5791                                              struct amdgpu_irq_src *source,
5792                                              unsigned type,
5793                                              enum amdgpu_interrupt_state state)
5794 {
5795         switch (state) {
5796         case AMDGPU_IRQ_STATE_DISABLE:
5797         case AMDGPU_IRQ_STATE_ENABLE:
5798                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5799                                PRIV_REG_INT_ENABLE,
5800                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5801                 break;
5802         default:
5803                 break;
5804         }
5805
5806         return 0;
5807 }
5808
5809 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5810                                               struct amdgpu_irq_src *source,
5811                                               unsigned type,
5812                                               enum amdgpu_interrupt_state state)
5813 {
5814         switch (state) {
5815         case AMDGPU_IRQ_STATE_DISABLE:
5816         case AMDGPU_IRQ_STATE_ENABLE:
5817                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5818                                PRIV_INSTR_INT_ENABLE,
5819                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5820                 break;
5821         default:
5822                 break;
5823         }
5824
5825         return 0;
5826 }
5827
5828 #define ENABLE_ECC_ON_ME_PIPE(me, pipe)                         \
5829         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5830                         CP_ECC_ERROR_INT_ENABLE, 1)
5831
5832 #define DISABLE_ECC_ON_ME_PIPE(me, pipe)                        \
5833         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5834                         CP_ECC_ERROR_INT_ENABLE, 0)
5835
5836 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5837                                               struct amdgpu_irq_src *source,
5838                                               unsigned type,
5839                                               enum amdgpu_interrupt_state state)
5840 {
5841         switch (state) {
5842         case AMDGPU_IRQ_STATE_DISABLE:
5843                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5844                                 CP_ECC_ERROR_INT_ENABLE, 0);
5845                 DISABLE_ECC_ON_ME_PIPE(1, 0);
5846                 DISABLE_ECC_ON_ME_PIPE(1, 1);
5847                 DISABLE_ECC_ON_ME_PIPE(1, 2);
5848                 DISABLE_ECC_ON_ME_PIPE(1, 3);
5849                 break;
5850
5851         case AMDGPU_IRQ_STATE_ENABLE:
5852                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5853                                 CP_ECC_ERROR_INT_ENABLE, 1);
5854                 ENABLE_ECC_ON_ME_PIPE(1, 0);
5855                 ENABLE_ECC_ON_ME_PIPE(1, 1);
5856                 ENABLE_ECC_ON_ME_PIPE(1, 2);
5857                 ENABLE_ECC_ON_ME_PIPE(1, 3);
5858                 break;
5859         default:
5860                 break;
5861         }
5862
5863         return 0;
5864 }
5865
5866
5867 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5868                                             struct amdgpu_irq_src *src,
5869                                             unsigned type,
5870                                             enum amdgpu_interrupt_state state)
5871 {
5872         switch (type) {
5873         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5874                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5875                 break;
5876         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5877                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5878                 break;
5879         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5880                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5881                 break;
5882         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5883                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5884                 break;
5885         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5886                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5887                 break;
5888         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5889                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5890                 break;
5891         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5892                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5893                 break;
5894         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5895                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5896                 break;
5897         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5898                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5899                 break;
5900         default:
5901                 break;
5902         }
5903         return 0;
5904 }
5905
5906 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5907                             struct amdgpu_irq_src *source,
5908                             struct amdgpu_iv_entry *entry)
5909 {
5910         int i;
5911         u8 me_id, pipe_id, queue_id;
5912         struct amdgpu_ring *ring;
5913
5914         DRM_DEBUG("IH: CP EOP\n");
5915         me_id = (entry->ring_id & 0x0c) >> 2;
5916         pipe_id = (entry->ring_id & 0x03) >> 0;
5917         queue_id = (entry->ring_id & 0x70) >> 4;
5918
5919         switch (me_id) {
5920         case 0:
5921                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5922                 break;
5923         case 1:
5924         case 2:
5925                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5926                         ring = &adev->gfx.compute_ring[i];
5927                         /* Per-queue interrupt is supported for MEC starting from VI.
5928                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
5929                           */
5930                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5931                                 amdgpu_fence_process(ring);
5932                 }
5933                 break;
5934         }
5935         return 0;
5936 }
5937
5938 static void gfx_v9_0_fault(struct amdgpu_device *adev,
5939                            struct amdgpu_iv_entry *entry)
5940 {
5941         u8 me_id, pipe_id, queue_id;
5942         struct amdgpu_ring *ring;
5943         int i;
5944
5945         me_id = (entry->ring_id & 0x0c) >> 2;
5946         pipe_id = (entry->ring_id & 0x03) >> 0;
5947         queue_id = (entry->ring_id & 0x70) >> 4;
5948
5949         switch (me_id) {
5950         case 0:
5951                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5952                 break;
5953         case 1:
5954         case 2:
5955                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5956                         ring = &adev->gfx.compute_ring[i];
5957                         if (ring->me == me_id && ring->pipe == pipe_id &&
5958                             ring->queue == queue_id)
5959                                 drm_sched_fault(&ring->sched);
5960                 }
5961                 break;
5962         }
5963 }
5964
5965 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5966                                  struct amdgpu_irq_src *source,
5967                                  struct amdgpu_iv_entry *entry)
5968 {
5969         DRM_ERROR("Illegal register access in command stream\n");
5970         gfx_v9_0_fault(adev, entry);
5971         return 0;
5972 }
5973
5974 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5975                                   struct amdgpu_irq_src *source,
5976                                   struct amdgpu_iv_entry *entry)
5977 {
5978         DRM_ERROR("Illegal instruction in command stream\n");
5979         gfx_v9_0_fault(adev, entry);
5980         return 0;
5981 }
5982
5983
5984 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5985         { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5986           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5987           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5988         },
5989         { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5990           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5991           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5992         },
5993         { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5994           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5995           0, 0
5996         },
5997         { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5998           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5999           0, 0
6000         },
6001         { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
6002           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
6003           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
6004         },
6005         { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6006           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
6007           0, 0
6008         },
6009         { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6010           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
6011           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
6012         },
6013         { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
6014           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
6015           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
6016         },
6017         { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6018           SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
6019           0, 0
6020         },
6021         { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6022           SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
6023           0, 0
6024         },
6025         { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6026           SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
6027           0, 0
6028         },
6029         { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6030           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
6031           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
6032         },
6033         { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6034           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
6035           0, 0
6036         },
6037         { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6038           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6039           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6040         },
6041         { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6042           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6043           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6044           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6045         },
6046         { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6047           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6048           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6049           0, 0
6050         },
6051         { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6052           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6053           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6054           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6055         },
6056         { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6057           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6058           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6059           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6060         },
6061         { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6062           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6063           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6064           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6065         },
6066         { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6067           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6068           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6069           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6070         },
6071         { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6072           SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6073           0, 0
6074         },
6075         { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6076           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6077           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6078         },
6079         { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6080           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6081           0, 0
6082         },
6083         { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6084           SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6085           0, 0
6086         },
6087         { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6088           SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6089           0, 0
6090         },
6091         { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6092           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6093           0, 0
6094         },
6095         { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6096           SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6097           0, 0
6098         },
6099         { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6100           SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6101           0, 0
6102         },
6103         { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6104           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6105           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6106         },
6107         { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6108           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6109           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6110         },
6111         { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6112           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6113           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6114         },
6115         { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6116           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6117           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6118         },
6119         { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6120           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6121           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6122         },
6123         { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6124           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6125           0, 0
6126         },
6127         { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6128           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6129           0, 0
6130         },
6131         { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6132           SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6133           0, 0
6134         },
6135         { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6136           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6137           0, 0
6138         },
6139         { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6140           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6141           0, 0
6142         },
6143         { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6144           SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6145           0, 0
6146         },
6147         { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6148           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6149           0, 0
6150         },
6151         { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6152           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6153           0, 0
6154         },
6155         { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6156           SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6157           0, 0
6158         },
6159         { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6160           SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6161           0, 0
6162         },
6163         { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6164           SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6165           0, 0
6166         },
6167         { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6168           SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6169           0, 0
6170         },
6171         { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6172           SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6173           0, 0
6174         },
6175         { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6176           SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6177           0, 0
6178         },
6179         { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6180           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6181           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6182         },
6183         { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6184           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6185           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6186         },
6187         { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6188           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6189           0, 0
6190         },
6191         { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6192           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6193           0, 0
6194         },
6195         { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6196           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6197           0, 0
6198         },
6199         { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6200           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6201           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6202         },
6203         { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6204           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6205           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6206         },
6207         { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6208           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6209           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6210         },
6211         { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6212           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6213           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6214         },
6215         { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6216           SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6217           0, 0
6218         },
6219         { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6220           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6221           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6222         },
6223         { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6224           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6225           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6226         },
6227         { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6228           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6229           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6230         },
6231         { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6232           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6233           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6234         },
6235         { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6236           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6237           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6238         },
6239         { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6240           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6241           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6242         },
6243         { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6244           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6245           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6246         },
6247         { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6248           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6249           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6250         },
6251         { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6252           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6253           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6254         },
6255         { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6256           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6257           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6258         },
6259         { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6260           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6261           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6262         },
6263         { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6264           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6265           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6266         },
6267         { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6268           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6269           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6270         },
6271         { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6272           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6273           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6274         },
6275         { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6276           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6277           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6278         },
6279         { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6280           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6281           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6282         },
6283         { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6284           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6285           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6286         },
6287         { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6288           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6289           0, 0
6290         },
6291         { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6292           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6293           0, 0
6294         },
6295         { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6296           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6297           0, 0
6298         },
6299         { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6300           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6301           0, 0
6302         },
6303         { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6304           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6305           0, 0
6306         },
6307         { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6308           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6309           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6310         },
6311         { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6312           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6313           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6314         },
6315         { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6316           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6317           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6318         },
6319         { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6320           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6321           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6322         },
6323         { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6324           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6325           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6326         },
6327         { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6328           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6329           0, 0
6330         },
6331         { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6332           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6333           0, 0
6334         },
6335         { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6336           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6337           0, 0
6338         },
6339         { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6340           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6341           0, 0
6342         },
6343         { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6344           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6345           0, 0
6346         },
6347         { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6348           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6349           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6350         },
6351         { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6352           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6353           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6354         },
6355         { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6356           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6357           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6358         },
6359         { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6360           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6361           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6362         },
6363         { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6364           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6365           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6366         },
6367         { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6368           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6369           0, 0
6370         },
6371         { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6372           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6373           0, 0
6374         },
6375         { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6376           SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6377           0, 0
6378         },
6379         { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6380           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6381           0, 0
6382         },
6383         { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6384           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6385           0, 0
6386         },
6387         { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6388           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6389           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6390         },
6391         { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6392           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6393           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6394         },
6395         { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6396           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6397           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6398         },
6399         { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6400           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6401           0, 0
6402         },
6403         { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6404           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6405           0, 0
6406         },
6407         { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6408           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6409           0, 0
6410         },
6411         { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6412           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6413           0, 0
6414         },
6415         { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6416           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6417           0, 0
6418         },
6419         { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6420           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6421           0, 0
6422         }
6423 };
6424
6425 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6426                                      void *inject_if)
6427 {
6428         struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6429         int ret;
6430         struct ta_ras_trigger_error_input block_info = { 0 };
6431
6432         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6433                 return -EINVAL;
6434
6435         if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6436                 return -EINVAL;
6437
6438         if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6439                 return -EPERM;
6440
6441         if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6442               info->head.type)) {
6443                 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6444                         ras_gfx_subblocks[info->head.sub_block_index].name,
6445                         info->head.type);
6446                 return -EPERM;
6447         }
6448
6449         if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6450               info->head.type)) {
6451                 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6452                         ras_gfx_subblocks[info->head.sub_block_index].name,
6453                         info->head.type);
6454                 return -EPERM;
6455         }
6456
6457         block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6458         block_info.sub_block_index =
6459                 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6460         block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6461         block_info.address = info->address;
6462         block_info.value = info->value;
6463
6464         mutex_lock(&adev->grbm_idx_mutex);
6465         ret = psp_ras_trigger_error(&adev->psp, &block_info);
6466         mutex_unlock(&adev->grbm_idx_mutex);
6467
6468         return ret;
6469 }
6470
6471 static const char *vml2_mems[] = {
6472         "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6473         "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6474         "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6475         "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6476         "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6477         "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6478         "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6479         "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6480         "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6481         "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6482         "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6483         "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6484         "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6485         "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6486         "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6487         "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6488 };
6489
6490 static const char *vml2_walker_mems[] = {
6491         "UTC_VML2_CACHE_PDE0_MEM0",
6492         "UTC_VML2_CACHE_PDE0_MEM1",
6493         "UTC_VML2_CACHE_PDE1_MEM0",
6494         "UTC_VML2_CACHE_PDE1_MEM1",
6495         "UTC_VML2_CACHE_PDE2_MEM0",
6496         "UTC_VML2_CACHE_PDE2_MEM1",
6497         "UTC_VML2_RDIF_LOG_FIFO",
6498 };
6499
6500 static const char *atc_l2_cache_2m_mems[] = {
6501         "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6502         "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6503         "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6504         "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6505 };
6506
6507 static const char *atc_l2_cache_4k_mems[] = {
6508         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6509         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6510         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6511         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6512         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6513         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6514         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6515         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6516         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6517         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6518         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6519         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6520         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6521         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6522         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6523         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6524         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6525         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6526         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6527         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6528         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6529         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6530         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6531         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6532         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6533         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6534         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6535         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6536         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6537         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6538         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6539         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6540 };
6541
6542 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6543                                          struct ras_err_data *err_data)
6544 {
6545         uint32_t i, data;
6546         uint32_t sec_count, ded_count;
6547
6548         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6549         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6550         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6551         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6552         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6553         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6554         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6555         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6556
6557         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6558                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6559                 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6560
6561                 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6562                 if (sec_count) {
6563                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6564                                 "SEC %d\n", i, vml2_mems[i], sec_count);
6565                         err_data->ce_count += sec_count;
6566                 }
6567
6568                 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6569                 if (ded_count) {
6570                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6571                                 "DED %d\n", i, vml2_mems[i], ded_count);
6572                         err_data->ue_count += ded_count;
6573                 }
6574         }
6575
6576         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6577                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6578                 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6579
6580                 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6581                                                 SEC_COUNT);
6582                 if (sec_count) {
6583                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6584                                 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6585                         err_data->ce_count += sec_count;
6586                 }
6587
6588                 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6589                                                 DED_COUNT);
6590                 if (ded_count) {
6591                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6592                                 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6593                         err_data->ue_count += ded_count;
6594                 }
6595         }
6596
6597         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6598                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6599                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6600
6601                 sec_count = (data & 0x00006000L) >> 0xd;
6602                 if (sec_count) {
6603                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6604                                 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6605                                 sec_count);
6606                         err_data->ce_count += sec_count;
6607                 }
6608         }
6609
6610         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6611                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6612                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6613
6614                 sec_count = (data & 0x00006000L) >> 0xd;
6615                 if (sec_count) {
6616                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6617                                 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6618                                 sec_count);
6619                         err_data->ce_count += sec_count;
6620                 }
6621
6622                 ded_count = (data & 0x00018000L) >> 0xf;
6623                 if (ded_count) {
6624                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6625                                 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6626                                 ded_count);
6627                         err_data->ue_count += ded_count;
6628                 }
6629         }
6630
6631         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6632         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6633         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6634         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6635
6636         return 0;
6637 }
6638
6639 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6640         const struct soc15_reg_entry *reg,
6641         uint32_t se_id, uint32_t inst_id, uint32_t value,
6642         uint32_t *sec_count, uint32_t *ded_count)
6643 {
6644         uint32_t i;
6645         uint32_t sec_cnt, ded_cnt;
6646
6647         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6648                 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6649                         gfx_v9_0_ras_fields[i].seg != reg->seg ||
6650                         gfx_v9_0_ras_fields[i].inst != reg->inst)
6651                         continue;
6652
6653                 sec_cnt = (value &
6654                                 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6655                                 gfx_v9_0_ras_fields[i].sec_count_shift;
6656                 if (sec_cnt) {
6657                         dev_info(adev->dev, "GFX SubBlock %s, "
6658                                 "Instance[%d][%d], SEC %d\n",
6659                                 gfx_v9_0_ras_fields[i].name,
6660                                 se_id, inst_id,
6661                                 sec_cnt);
6662                         *sec_count += sec_cnt;
6663                 }
6664
6665                 ded_cnt = (value &
6666                                 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6667                                 gfx_v9_0_ras_fields[i].ded_count_shift;
6668                 if (ded_cnt) {
6669                         dev_info(adev->dev, "GFX SubBlock %s, "
6670                                 "Instance[%d][%d], DED %d\n",
6671                                 gfx_v9_0_ras_fields[i].name,
6672                                 se_id, inst_id,
6673                                 ded_cnt);
6674                         *ded_count += ded_cnt;
6675                 }
6676         }
6677
6678         return 0;
6679 }
6680
6681 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6682 {
6683         int i, j, k;
6684
6685         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6686                 return;
6687
6688         /* read back registers to clear the counters */
6689         mutex_lock(&adev->grbm_idx_mutex);
6690         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6691                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6692                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6693                                 gfx_v9_0_select_se_sh(adev, j, 0x0, k);
6694                                 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6695                         }
6696                 }
6697         }
6698         WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6699         mutex_unlock(&adev->grbm_idx_mutex);
6700
6701         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6702         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6703         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6704         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6705         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6706         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6707         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6708         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6709
6710         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6711                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6712                 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6713         }
6714
6715         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6716                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6717                 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6718         }
6719
6720         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6721                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6722                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6723         }
6724
6725         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6726                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6727                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6728         }
6729
6730         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6731         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6732         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6733         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6734 }
6735
6736 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6737                                           void *ras_error_status)
6738 {
6739         struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6740         uint32_t sec_count = 0, ded_count = 0;
6741         uint32_t i, j, k;
6742         uint32_t reg_value;
6743
6744         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6745                 return -EINVAL;
6746
6747         err_data->ue_count = 0;
6748         err_data->ce_count = 0;
6749
6750         mutex_lock(&adev->grbm_idx_mutex);
6751
6752         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6753                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6754                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6755                                 gfx_v9_0_select_se_sh(adev, j, 0, k);
6756                                 reg_value =
6757                                         RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6758                                 if (reg_value)
6759                                         gfx_v9_0_ras_error_count(adev,
6760                                                 &gfx_v9_0_edc_counter_regs[i],
6761                                                 j, k, reg_value,
6762                                                 &sec_count, &ded_count);
6763                         }
6764                 }
6765         }
6766
6767         err_data->ce_count += sec_count;
6768         err_data->ue_count += ded_count;
6769
6770         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6771         mutex_unlock(&adev->grbm_idx_mutex);
6772
6773         gfx_v9_0_query_utc_edc_status(adev, err_data);
6774
6775         return 0;
6776 }
6777
6778 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6779 {
6780         const unsigned int cp_coher_cntl =
6781                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6782                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6783                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6784                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6785                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6786
6787         /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
6788         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6789         amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
6790         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6791         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6792         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6793         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6794         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6795 }
6796
6797 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6798                                         uint32_t pipe, bool enable)
6799 {
6800         struct amdgpu_device *adev = ring->adev;
6801         uint32_t val;
6802         uint32_t wcl_cs_reg;
6803
6804         /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6805         val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
6806
6807         switch (pipe) {
6808         case 0:
6809                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
6810                 break;
6811         case 1:
6812                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
6813                 break;
6814         case 2:
6815                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
6816                 break;
6817         case 3:
6818                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
6819                 break;
6820         default:
6821                 DRM_DEBUG("invalid pipe %d\n", pipe);
6822                 return;
6823         }
6824
6825         amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6826
6827 }
6828 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6829 {
6830         struct amdgpu_device *adev = ring->adev;
6831         uint32_t val;
6832         int i;
6833
6834
6835         /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6836          * number of gfx waves. Setting 5 bit will make sure gfx only gets
6837          * around 25% of gpu resources.
6838          */
6839         val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6840         amdgpu_ring_emit_wreg(ring,
6841                               SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
6842                               val);
6843
6844         /* Restrict waves for normal/low priority compute queues as well
6845          * to get best QoS for high priority compute jobs.
6846          *
6847          * amdgpu controls only 1st ME(0-3 CS pipes).
6848          */
6849         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6850                 if (i != ring->pipe)
6851                         gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
6852
6853         }
6854 }
6855
6856 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6857         .name = "gfx_v9_0",
6858         .early_init = gfx_v9_0_early_init,
6859         .late_init = gfx_v9_0_late_init,
6860         .sw_init = gfx_v9_0_sw_init,
6861         .sw_fini = gfx_v9_0_sw_fini,
6862         .hw_init = gfx_v9_0_hw_init,
6863         .hw_fini = gfx_v9_0_hw_fini,
6864         .suspend = gfx_v9_0_suspend,
6865         .resume = gfx_v9_0_resume,
6866         .is_idle = gfx_v9_0_is_idle,
6867         .wait_for_idle = gfx_v9_0_wait_for_idle,
6868         .soft_reset = gfx_v9_0_soft_reset,
6869         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6870         .set_powergating_state = gfx_v9_0_set_powergating_state,
6871         .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6872 };
6873
6874 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6875         .type = AMDGPU_RING_TYPE_GFX,
6876         .align_mask = 0xff,
6877         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6878         .support_64bit_ptrs = true,
6879         .vmhub = AMDGPU_GFXHUB_0,
6880         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6881         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6882         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6883         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6884                 5 +  /* COND_EXEC */
6885                 7 +  /* PIPELINE_SYNC */
6886                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6887                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6888                 2 + /* VM_FLUSH */
6889                 8 +  /* FENCE for VM_FLUSH */
6890                 20 + /* GDS switch */
6891                 4 + /* double SWITCH_BUFFER,
6892                        the first COND_EXEC jump to the place just
6893                            prior to this double SWITCH_BUFFER  */
6894                 5 + /* COND_EXEC */
6895                 7 +      /*     HDP_flush */
6896                 4 +      /*     VGT_flush */
6897                 14 + /* CE_META */
6898                 31 + /* DE_META */
6899                 3 + /* CNTX_CTRL */
6900                 5 + /* HDP_INVL */
6901                 8 + 8 + /* FENCE x2 */
6902                 2 + /* SWITCH_BUFFER */
6903                 7, /* gfx_v9_0_emit_mem_sync */
6904         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6905         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6906         .emit_fence = gfx_v9_0_ring_emit_fence,
6907         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6908         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6909         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6910         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6911         .test_ring = gfx_v9_0_ring_test_ring,
6912         .test_ib = gfx_v9_0_ring_test_ib,
6913         .insert_nop = amdgpu_ring_insert_nop,
6914         .pad_ib = amdgpu_ring_generic_pad_ib,
6915         .emit_switch_buffer = gfx_v9_ring_emit_sb,
6916         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6917         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6918         .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6919         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6920         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6921         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6922         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6923         .soft_recovery = gfx_v9_0_ring_soft_recovery,
6924         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6925 };
6926
6927 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6928         .type = AMDGPU_RING_TYPE_COMPUTE,
6929         .align_mask = 0xff,
6930         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6931         .support_64bit_ptrs = true,
6932         .vmhub = AMDGPU_GFXHUB_0,
6933         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6934         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6935         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6936         .emit_frame_size =
6937                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6938                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6939                 5 + /* hdp invalidate */
6940                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6941                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6942                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6943                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6944                 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6945                 7 + /* gfx_v9_0_emit_mem_sync */
6946                 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6947                 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6948         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6949         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6950         .emit_fence = gfx_v9_0_ring_emit_fence,
6951         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6952         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6953         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6954         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6955         .test_ring = gfx_v9_0_ring_test_ring,
6956         .test_ib = gfx_v9_0_ring_test_ib,
6957         .insert_nop = amdgpu_ring_insert_nop,
6958         .pad_ib = amdgpu_ring_generic_pad_ib,
6959         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6960         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6961         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6962         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6963         .emit_wave_limit = gfx_v9_0_emit_wave_limit,
6964 };
6965
6966 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6967         .type = AMDGPU_RING_TYPE_KIQ,
6968         .align_mask = 0xff,
6969         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6970         .support_64bit_ptrs = true,
6971         .vmhub = AMDGPU_GFXHUB_0,
6972         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6973         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6974         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6975         .emit_frame_size =
6976                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6977                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6978                 5 + /* hdp invalidate */
6979                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6980                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6981                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6982                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6983                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6984         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6985         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6986         .test_ring = gfx_v9_0_ring_test_ring,
6987         .insert_nop = amdgpu_ring_insert_nop,
6988         .pad_ib = amdgpu_ring_generic_pad_ib,
6989         .emit_rreg = gfx_v9_0_ring_emit_rreg,
6990         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6991         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6992         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6993 };
6994
6995 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6996 {
6997         int i;
6998
6999         adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
7000
7001         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7002                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
7003
7004         for (i = 0; i < adev->gfx.num_compute_rings; i++)
7005                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
7006 }
7007
7008 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
7009         .set = gfx_v9_0_set_eop_interrupt_state,
7010         .process = gfx_v9_0_eop_irq,
7011 };
7012
7013 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
7014         .set = gfx_v9_0_set_priv_reg_fault_state,
7015         .process = gfx_v9_0_priv_reg_irq,
7016 };
7017
7018 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
7019         .set = gfx_v9_0_set_priv_inst_fault_state,
7020         .process = gfx_v9_0_priv_inst_irq,
7021 };
7022
7023 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
7024         .set = gfx_v9_0_set_cp_ecc_error_state,
7025         .process = amdgpu_gfx_cp_ecc_error_irq,
7026 };
7027
7028
7029 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
7030 {
7031         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7032         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
7033
7034         adev->gfx.priv_reg_irq.num_types = 1;
7035         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
7036
7037         adev->gfx.priv_inst_irq.num_types = 1;
7038         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7039
7040         adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7041         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7042 }
7043
7044 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7045 {
7046         switch (adev->ip_versions[GC_HWIP][0]) {
7047         case IP_VERSION(9, 0, 1):
7048         case IP_VERSION(9, 2, 1):
7049         case IP_VERSION(9, 4, 0):
7050         case IP_VERSION(9, 2, 2):
7051         case IP_VERSION(9, 1, 0):
7052         case IP_VERSION(9, 4, 1):
7053         case IP_VERSION(9, 3, 0):
7054         case IP_VERSION(9, 4, 2):
7055                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7056                 break;
7057         default:
7058                 break;
7059         }
7060 }
7061
7062 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7063 {
7064         /* init asci gds info */
7065         switch (adev->ip_versions[GC_HWIP][0]) {
7066         case IP_VERSION(9, 0, 1):
7067         case IP_VERSION(9, 2, 1):
7068         case IP_VERSION(9, 4, 0):
7069                 adev->gds.gds_size = 0x10000;
7070                 break;
7071         case IP_VERSION(9, 2, 2):
7072         case IP_VERSION(9, 1, 0):
7073         case IP_VERSION(9, 4, 1):
7074                 adev->gds.gds_size = 0x1000;
7075                 break;
7076         case IP_VERSION(9, 4, 2):
7077                 /* aldebaran removed all the GDS internal memory,
7078                  * only support GWS opcode in kernel, like barrier
7079                  * semaphore.etc */
7080                 adev->gds.gds_size = 0;
7081                 break;
7082         default:
7083                 adev->gds.gds_size = 0x10000;
7084                 break;
7085         }
7086
7087         switch (adev->ip_versions[GC_HWIP][0]) {
7088         case IP_VERSION(9, 0, 1):
7089         case IP_VERSION(9, 4, 0):
7090                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7091                 break;
7092         case IP_VERSION(9, 2, 1):
7093                 adev->gds.gds_compute_max_wave_id = 0x27f;
7094                 break;
7095         case IP_VERSION(9, 2, 2):
7096         case IP_VERSION(9, 1, 0):
7097                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7098                         adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7099                 else
7100                         adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7101                 break;
7102         case IP_VERSION(9, 4, 1):
7103                 adev->gds.gds_compute_max_wave_id = 0xfff;
7104                 break;
7105         case IP_VERSION(9, 4, 2):
7106                 /* deprecated for Aldebaran, no usage at all */
7107                 adev->gds.gds_compute_max_wave_id = 0;
7108                 break;
7109         default:
7110                 /* this really depends on the chip */
7111                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7112                 break;
7113         }
7114
7115         adev->gds.gws_size = 64;
7116         adev->gds.oa_size = 16;
7117 }
7118
7119 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7120                                                  u32 bitmap)
7121 {
7122         u32 data;
7123
7124         if (!bitmap)
7125                 return;
7126
7127         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7128         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7129
7130         WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7131 }
7132
7133 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7134 {
7135         u32 data, mask;
7136
7137         data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7138         data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7139
7140         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7141         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7142
7143         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7144
7145         return (~data) & mask;
7146 }
7147
7148 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7149                                  struct amdgpu_cu_info *cu_info)
7150 {
7151         int i, j, k, counter, active_cu_number = 0;
7152         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7153         unsigned disable_masks[4 * 4];
7154
7155         if (!adev || !cu_info)
7156                 return -EINVAL;
7157
7158         /*
7159          * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7160          */
7161         if (adev->gfx.config.max_shader_engines *
7162                 adev->gfx.config.max_sh_per_se > 16)
7163                 return -EINVAL;
7164
7165         amdgpu_gfx_parse_disable_cu(disable_masks,
7166                                     adev->gfx.config.max_shader_engines,
7167                                     adev->gfx.config.max_sh_per_se);
7168
7169         mutex_lock(&adev->grbm_idx_mutex);
7170         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7171                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7172                         mask = 1;
7173                         ao_bitmap = 0;
7174                         counter = 0;
7175                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
7176                         gfx_v9_0_set_user_cu_inactive_bitmap(
7177                                 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7178                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7179
7180                         /*
7181                          * The bitmap(and ao_cu_bitmap) in cu_info structure is
7182                          * 4x4 size array, and it's usually suitable for Vega
7183                          * ASICs which has 4*2 SE/SH layout.
7184                          * But for Arcturus, SE/SH layout is changed to 8*1.
7185                          * To mostly reduce the impact, we make it compatible
7186                          * with current bitmap array as below:
7187                          *    SE4,SH0 --> bitmap[0][1]
7188                          *    SE5,SH0 --> bitmap[1][1]
7189                          *    SE6,SH0 --> bitmap[2][1]
7190                          *    SE7,SH0 --> bitmap[3][1]
7191                          */
7192                         cu_info->bitmap[i % 4][j + i / 4] = bitmap;
7193
7194                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7195                                 if (bitmap & mask) {
7196                                         if (counter < adev->gfx.config.max_cu_per_sh)
7197                                                 ao_bitmap |= mask;
7198                                         counter ++;
7199                                 }
7200                                 mask <<= 1;
7201                         }
7202                         active_cu_number += counter;
7203                         if (i < 2 && j < 2)
7204                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7205                         cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7206                 }
7207         }
7208         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7209         mutex_unlock(&adev->grbm_idx_mutex);
7210
7211         cu_info->number = active_cu_number;
7212         cu_info->ao_cu_mask = ao_cu_mask;
7213         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7214
7215         return 0;
7216 }
7217
7218 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7219 {
7220         .type = AMD_IP_BLOCK_TYPE_GFX,
7221         .major = 9,
7222         .minor = 0,
7223         .rev = 0,
7224         .funcs = &gfx_v9_0_ip_funcs,
7225 };