drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "soc15.h"
  33 #include "soc15d.h"
  34 #include "amdgpu_atomfirmware.h"
  35 #include "amdgpu_pm.h"
  36
  37 #include "gc/gc_9_0_offset.h"
  38 #include "gc/gc_9_0_sh_mask.h"
  39
  40 #include "vega10_enum.h"
  41
  42 #include "soc15_common.h"
  43 #include "clearstate_gfx9.h"
  44 #include "v9_structs.h"
  45
  46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
  47
  48 #include "amdgpu_ras.h"
  49
  50 #include "gfx_v9_4.h"
  51 #include "gfx_v9_0.h"
  52
  53 #include "asic_reg/pwr/pwr_10_0_offset.h"
  54 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
  55 #include "asic_reg/gc/gc_9_0_default.h"
  56
  57 #define GFX9_NUM_GFX_RINGS     1
  58 #define GFX9_MEC_HPD_SIZE 4096
  59 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
  60 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
  61
  62 #define mmGCEA_PROBE_MAP                        0x070c
  63 #define mmGCEA_PROBE_MAP_BASE_IDX               0
  64
  65 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  66 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  67 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  68 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  69 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  70 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  71
  72 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
  73 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
  74 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
  75 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
  76 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
  77 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
  78
  79 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
  80 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
  81 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
  82 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
  83 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
  84 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
  85
  86 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
  87 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
  88 MODULE_FIRMWARE("amdgpu/raven_me.bin");
  89 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
  91 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
  92
  93 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
  94 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
  95 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
  96 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
  97 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
  98 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
  99 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
 100
 101 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
 102 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
 103 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
 104 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
 105 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
 106 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
 107 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
 108
 109 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
 110 MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin");
 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
 112
 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
 117 MODULE_FIRMWARE("amdgpu/renoir_mec2.bin");
 118 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
 119
 120 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
 121 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
 122 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
 124 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
 125 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
 126
 127 #define mmTCP_CHAN_STEER_0_ARCT                                                         0x0b03
 128 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX                                                        0
 129 #define mmTCP_CHAN_STEER_1_ARCT                                                         0x0b04
 130 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX                                                        0
 131 #define mmTCP_CHAN_STEER_2_ARCT                                                         0x0b09
 132 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX                                                        0
 133 #define mmTCP_CHAN_STEER_3_ARCT                                                         0x0b0a
 134 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX                                                        0
 135 #define mmTCP_CHAN_STEER_4_ARCT                                                         0x0b0b
 136 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX                                                        0
 137 #define mmTCP_CHAN_STEER_5_ARCT                                                         0x0b0c
 138 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX                                                        0
 139
 140 enum ta_ras_gfx_subblock {
 141         /*CPC*/
 142         TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
 143         TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
 144         TA_RAS_BLOCK__GFX_CPC_UCODE,
 145         TA_RAS_BLOCK__GFX_DC_STATE_ME1,
 146         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
 147         TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
 148         TA_RAS_BLOCK__GFX_DC_STATE_ME2,
 149         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
 150         TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 151         TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 152         /* CPF*/
 153         TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 154         TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 155         TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
 156         TA_RAS_BLOCK__GFX_CPF_TAG,
 157         TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
 158         /* CPG*/
 159         TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 160         TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 161         TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
 162         TA_RAS_BLOCK__GFX_CPG_TAG,
 163         TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
 164         /* GDS*/
 165         TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 166         TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 167         TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
 168         TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
 169         TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
 170         TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 171         TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 172         /* SPI*/
 173         TA_RAS_BLOCK__GFX_SPI_SR_MEM,
 174         /* SQ*/
 175         TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 176         TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 177         TA_RAS_BLOCK__GFX_SQ_LDS_D,
 178         TA_RAS_BLOCK__GFX_SQ_LDS_I,
 179         TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
 180         TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
 181         /* SQC (3 ranges)*/
 182         TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 183         /* SQC range 0*/
 184         TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 185         TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
 186                 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
 187         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
 188         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
 189         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
 190         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
 191         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
 192         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 193         TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
 194                 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 195         /* SQC range 1*/
 196         TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 197         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
 198                 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 199         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
 200         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
 201         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
 202         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
 203         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
 204         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
 205         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
 206         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 207         TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
 208                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 209         /* SQC range 2*/
 210         TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 211         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
 212                 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 213         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
 214         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
 215         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
 216         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
 217         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
 218         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
 219         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
 220         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 221         TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
 222                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 223         TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
 224         /* TA*/
 225         TA_RAS_BLOCK__GFX_TA_INDEX_START,
 226         TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
 227         TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
 228         TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
 229         TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
 230         TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 231         TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 232         /* TCA*/
 233         TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 234         TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 235         TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 236         TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 237         /* TCC (5 sub-ranges)*/
 238         TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 239         /* TCC range 0*/
 240         TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 241         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
 242         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
 243         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
 244         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
 245         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
 246         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
 247         TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
 248         TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 249         TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 250         /* TCC range 1*/
 251         TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 252         TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 253         TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 254         TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
 255                 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 256         /* TCC range 2*/
 257         TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 258         TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 259         TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
 260         TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
 261         TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
 262         TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
 263         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
 264         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
 265         TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 266         TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
 267                 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 268         /* TCC range 3*/
 269         TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 270         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 271         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 272         TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
 273                 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 274         /* TCC range 4*/
 275         TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 276         TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
 277                 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 278         TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 279         TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
 280                 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 281         TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
 282         /* TCI*/
 283         TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
 284         /* TCP*/
 285         TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 286         TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 287         TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
 288         TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
 289         TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
 290         TA_RAS_BLOCK__GFX_TCP_DB_RAM,
 291         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
 292         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 293         TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 294         /* TD*/
 295         TA_RAS_BLOCK__GFX_TD_INDEX_START,
 296         TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
 297         TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
 298         TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 299         TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 300         /* EA (3 sub-ranges)*/
 301         TA_RAS_BLOCK__GFX_EA_INDEX_START,
 302         /* EA range 0*/
 303         TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
 304         TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
 305         TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
 306         TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
 307         TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
 308         TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
 309         TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
 310         TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
 311         TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 312         TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 313         /* EA range 1*/
 314         TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 315         TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 316         TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
 317         TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
 318         TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
 319         TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
 320         TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
 321         TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 322         TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 323         /* EA range 2*/
 324         TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 325         TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 326         TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
 327         TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
 328         TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 329         TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 330         TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
 331         /* UTC VM L2 bank*/
 332         TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
 333         /* UTC VM walker*/
 334         TA_RAS_BLOCK__UTC_VML2_WALKER,
 335         /* UTC ATC L2 2MB cache*/
 336         TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
 337         /* UTC ATC L2 4KB cache*/
 338         TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
 339         TA_RAS_BLOCK__GFX_MAX
 340 };
 341
 342 struct ras_gfx_subblock {
 343         unsigned char *name;
 344         int ta_subblock;
 345         int hw_supported_error_type;
 346         int sw_supported_error_type;
 347 };
 348
 349 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \
 350         [AMDGPU_RAS_BLOCK__##subblock] = {                                     \
 351                 #subblock,                                                     \
 352                 TA_RAS_BLOCK__##subblock,                                      \
 353                 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \
 354                 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \
 355         }
 356
 357 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
 358         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
 359         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
 360         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 361         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 362         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 363         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 364         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 365         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 366         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 367         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 368         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
 369         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
 370         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
 371         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
 372         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 373         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
 374         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
 375                              0),
 376         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
 377                              0),
 378         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 379         AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 380         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 381         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
 382         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
 383         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 384         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
 385         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 386                              0, 0),
 387         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 388                              0),
 389         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 390                              0, 0),
 391         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
 392                              0),
 393         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 394                              0, 0),
 395         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 396                              0),
 397         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 398                              1),
 399         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 400                              0, 0, 0),
 401         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 402                              0),
 403         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 404                              0),
 405         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 406                              0),
 407         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 408                              0),
 409         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 410                              0),
 411         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 412                              0, 0),
 413         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 414                              0),
 415         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 416                              0),
 417         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 418                              0, 0, 0),
 419         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 420                              0),
 421         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 422                              0),
 423         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 424                              0),
 425         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 426                              0),
 427         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 428                              0),
 429         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 430                              0, 0),
 431         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 432                              0),
 433         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
 434         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 435         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 436         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 437         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 438         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
 439         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 440         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
 441         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
 442                              1),
 443         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
 444                              1),
 445         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
 446                              1),
 447         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
 448                              0),
 449         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
 450                              0),
 451         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 452         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 453         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
 454         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
 455         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
 456         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
 457         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 458         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
 459         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
 460         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 461         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
 462         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
 463                              0),
 464         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 465         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
 466                              0),
 467         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
 468                              0, 0),
 469         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
 470                              0),
 471         AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 472         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
 473         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
 474         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 475         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 476         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 477         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
 478         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
 479         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
 480         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
 481         AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 482         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
 483         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 484         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 485         AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 486         AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 487         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 488         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 489         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 490         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 491         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 492         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 493         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 494         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 495         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 496         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 497         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 498         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 499         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 500         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 501         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
 502         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
 503         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
 504         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
 505 };
 506
 507 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
 508 {
 509         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 510         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
 511         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 512         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 513         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 514         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 515         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 516         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 517         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 518         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
 519         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
 520         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 521         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 522         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 523         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 524         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 525         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
 526         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 527         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 528         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 529 };
 530
 531 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 532 {
 533         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 534         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 535         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 536         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 537         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 538         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 539         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
 540         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 541         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
 542         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 543         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 544         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 545         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 546         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 547         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 548         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
 549         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
 550         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 551 };
 552
 553 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
 554 {
 555         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
 556         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 557         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 558         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
 559         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
 560         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
 561         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
 562         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
 563         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
 564         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
 565         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
 566 };
 567
 568 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 569 {
 570         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 571         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 572         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 573         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 574         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 575         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 576         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 577         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 578         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 579         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 580         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 581         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 582         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 583         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 584         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 585         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 586         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 587         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 588         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 589         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
 590         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
 591         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 592         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 593         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 594 };
 595
 596 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 597 {
 598         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 599         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
 600         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
 601         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
 602         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
 603         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 604         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 605 };
 606
 607 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
 608 {
 609         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
 610         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 611         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 612         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
 613         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
 614         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
 615         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
 616         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
 617         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 618         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 619         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
 620         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
 621         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
 622         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 623         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 624         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 625         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
 626         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 627         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 628 };
 629
 630 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
 631 {
 632         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 633         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 634         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 635         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
 636         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
 637         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 638         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 639         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 640         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 641         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 642         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 643         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
 644 };
 645
 646 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 647 {
 648         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
 649         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
 650         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 651 };
 652
 653 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
 654 {
 655         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 656         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 657         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 658         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 659         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 660         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 661         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 662         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 663         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
 664         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 665         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 666         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 667         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 668         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 669         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 670         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 671 };
 672
 673 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 674 {
 675         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
 676         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 677         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 678         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
 679         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
 680         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 681         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
 682         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 683         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
 684         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 685         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 686         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 687         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 688 };
 689
 690 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
 691 {
 692         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 693         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
 694         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
 695         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
 696         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
 697         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
 698         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
 699         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
 700         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
 701         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
 702         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
 703 };
 704
 705 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
 706         {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
 707         {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
 708 };
 709
 710 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
 711 {
 712         mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 713         mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 714         mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 715         mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 716         mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 717         mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 718         mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 719         mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 720 };
 721
 722 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 723 {
 724         mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 725         mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 726         mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 727         mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 728         mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 729         mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 730         mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 731         mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 732 };
 733
 734 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v)
 735 {
 736         static void *scratch_reg0;
 737         static void *scratch_reg1;
 738         static void *scratch_reg2;
 739         static void *scratch_reg3;
 740         static void *spare_int;
 741         static uint32_t grbm_cntl;
 742         static uint32_t grbm_idx;
 743
 744         scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4;
 745         scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4;
 746         scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4;
 747         scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4;
 748         spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4;
 749
 750         grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;
 751         grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;
 752
 753         if (amdgpu_sriov_runtime(adev)) {
 754                 pr_err("shouldn't call rlcg write register during runtime\n");
 755                 return;
 756         }
 757
 758         if (offset == grbm_cntl || offset == grbm_idx) {
 759                 if (offset  == grbm_cntl)
 760                         writel(v, scratch_reg2);
 761                 else if (offset == grbm_idx)
 762                         writel(v, scratch_reg3);
 763
 764                 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
 765         } else {
 766                 uint32_t i = 0;
 767                 uint32_t retries = 50000;
 768
 769                 writel(v, scratch_reg0);
 770                 writel(offset | 0x80000000, scratch_reg1);
 771                 writel(1, spare_int);
 772                 for (i = 0; i < retries; i++) {
 773                         u32 tmp;
 774
 775                         tmp = readl(scratch_reg1);
 776                         if (!(tmp & 0x80000000))
 777                                 break;
 778
 779                         udelay(10);
 780                 }
 781                 if (i >= retries)
 782                         pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
 783         }
 784
 785 }
 786
 787 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 788 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 789 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 790 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
 791
 792 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 793 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 794 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 795 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 796 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 797                                 struct amdgpu_cu_info *cu_info);
 798 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 799 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
 800 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
 801 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
 802                                           void *ras_error_status);
 803 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
 804                                      void *inject_if);
 805 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
 806
 807 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 808                                 uint64_t queue_mask)
 809 {
 810         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 811         amdgpu_ring_write(kiq_ring,
 812                 PACKET3_SET_RESOURCES_VMID_MASK(0) |
 813                 /* vmid_mask:0* queue_type:0 (KIQ) */
 814                 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
 815         amdgpu_ring_write(kiq_ring,
 816                         lower_32_bits(queue_mask));     /* queue mask lo */
 817         amdgpu_ring_write(kiq_ring,
 818                         upper_32_bits(queue_mask));     /* queue mask hi */
 819         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 820         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 821         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 822         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 823 }
 824
 825 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 826                                  struct amdgpu_ring *ring)
 827 {
 828         struct amdgpu_device *adev = kiq_ring->adev;
 829         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 830         uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
 831         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 832
 833         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 834         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 835         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 836                          PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 837                          PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 838                          PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 839                          PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 840                          PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
 841                          /*queue_type: normal compute queue */
 842                          PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
 843                          /* alloc format: all_on_one_pipe */
 844                          PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
 845                          PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 846                          /* num_queues: must be 1 */
 847                          PACKET3_MAP_QUEUES_NUM_QUEUES(1));
 848         amdgpu_ring_write(kiq_ring,
 849                         PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 850         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 851         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 852         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 853         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 854 }
 855
 856 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 857                                    struct amdgpu_ring *ring,
 858                                    enum amdgpu_unmap_queues_action action,
 859                                    u64 gpu_addr, u64 seq)
 860 {
 861         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 862
 863         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 864         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 865                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 866                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 867                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 868                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 869         amdgpu_ring_write(kiq_ring,
 870                         PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 871
 872         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 873                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 874                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 875                 amdgpu_ring_write(kiq_ring, seq);
 876         } else {
 877                 amdgpu_ring_write(kiq_ring, 0);
 878                 amdgpu_ring_write(kiq_ring, 0);
 879                 amdgpu_ring_write(kiq_ring, 0);
 880         }
 881 }
 882
 883 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 884                                    struct amdgpu_ring *ring,
 885                                    u64 addr,
 886                                    u64 seq)
 887 {
 888         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 889
 890         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 891         amdgpu_ring_write(kiq_ring,
 892                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 893                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 894                           PACKET3_QUERY_STATUS_COMMAND(2));
 895         /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 896         amdgpu_ring_write(kiq_ring,
 897                         PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 898                         PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 899         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 900         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 901         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 902         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 903 }
 904
 905 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 906                                 uint16_t pasid, uint32_t flush_type,
 907                                 bool all_hub)
 908 {
 909         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
 910         amdgpu_ring_write(kiq_ring,
 911                         PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
 912                         PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
 913                         PACKET3_INVALIDATE_TLBS_PASID(pasid) |
 914                         PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
 915 }
 916
 917 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
 918         .kiq_set_resources = gfx_v9_0_kiq_set_resources,
 919         .kiq_map_queues = gfx_v9_0_kiq_map_queues,
 920         .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
 921         .kiq_query_status = gfx_v9_0_kiq_query_status,
 922         .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
 923         .set_resources_size = 8,
 924         .map_queues_size = 7,
 925         .unmap_queues_size = 6,
 926         .query_status_size = 7,
 927         .invalidate_tlbs_size = 2,
 928 };
 929
 930 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 931 {
 932         adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
 933 }
 934
 935 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 936 {
 937         switch (adev->asic_type) {
 938         case CHIP_VEGA10:
 939                 soc15_program_register_sequence(adev,
 940                                                 golden_settings_gc_9_0,
 941                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 942                 soc15_program_register_sequence(adev,
 943                                                 golden_settings_gc_9_0_vg10,
 944                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
 945                 break;
 946         case CHIP_VEGA12:
 947                 soc15_program_register_sequence(adev,
 948                                                 golden_settings_gc_9_2_1,
 949                                                 ARRAY_SIZE(golden_settings_gc_9_2_1));
 950                 soc15_program_register_sequence(adev,
 951                                                 golden_settings_gc_9_2_1_vg12,
 952                                                 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
 953                 break;
 954         case CHIP_VEGA20:
 955                 soc15_program_register_sequence(adev,
 956                                                 golden_settings_gc_9_0,
 957                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 958                 soc15_program_register_sequence(adev,
 959                                                 golden_settings_gc_9_0_vg20,
 960                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
 961                 break;
 962         case CHIP_ARCTURUS:
 963                 soc15_program_register_sequence(adev,
 964                                                 golden_settings_gc_9_4_1_arct,
 965                                                 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
 966                 break;
 967         case CHIP_RAVEN:
 968                 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
 969                                                 ARRAY_SIZE(golden_settings_gc_9_1));
 970                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
 971                         soc15_program_register_sequence(adev,
 972                                                         golden_settings_gc_9_1_rv2,
 973                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv2));
 974                 else
 975                         soc15_program_register_sequence(adev,
 976                                                         golden_settings_gc_9_1_rv1,
 977                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv1));
 978                 break;
 979          case CHIP_RENOIR:
 980                 soc15_program_register_sequence(adev,
 981                                                 golden_settings_gc_9_1_rn,
 982                                                 ARRAY_SIZE(golden_settings_gc_9_1_rn));
 983                 return; /* for renoir, don't need common goldensetting */
 984         default:
 985                 break;
 986         }
 987
 988         if (adev->asic_type != CHIP_ARCTURUS)
 989                 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
 990                                                 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
 991 }
 992
 993 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
 994 {
 995         adev->gfx.scratch.num_reg = 8;
 996         adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
 997         adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 998 }
 999
1000 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1001                                        bool wc, uint32_t reg, uint32_t val)
1002 {
1003         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1004         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1005                                 WRITE_DATA_DST_SEL(0) |
1006                                 (wc ? WR_CONFIRM : 0));
1007         amdgpu_ring_write(ring, reg);
1008         amdgpu_ring_write(ring, 0);
1009         amdgpu_ring_write(ring, val);
1010 }
1011
1012 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1013                                   int mem_space, int opt, uint32_t addr0,
1014                                   uint32_t addr1, uint32_t ref, uint32_t mask,
1015                                   uint32_t inv)
1016 {
1017         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1018         amdgpu_ring_write(ring,
1019                                  /* memory (1) or register (0) */
1020                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1021                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
1022                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
1023                                  WAIT_REG_MEM_ENGINE(eng_sel)));
1024
1025         if (mem_space)
1026                 BUG_ON(addr0 & 0x3); /* Dword align */
1027         amdgpu_ring_write(ring, addr0);
1028         amdgpu_ring_write(ring, addr1);
1029         amdgpu_ring_write(ring, ref);
1030         amdgpu_ring_write(ring, mask);
1031         amdgpu_ring_write(ring, inv); /* poll interval */
1032 }
1033
1034 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1035 {
1036         struct amdgpu_device *adev = ring->adev;
1037         uint32_t scratch;
1038         uint32_t tmp = 0;
1039         unsigned i;
1040         int r;
1041
1042         r = amdgpu_gfx_scratch_get(adev, &scratch);
1043         if (r)
1044                 return r;
1045
1046         WREG32(scratch, 0xCAFEDEAD);
1047         r = amdgpu_ring_alloc(ring, 3);
1048         if (r)
1049                 goto error_free_scratch;
1050
1051         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1052         amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
1053         amdgpu_ring_write(ring, 0xDEADBEEF);
1054         amdgpu_ring_commit(ring);
1055
1056         for (i = 0; i < adev->usec_timeout; i++) {
1057                 tmp = RREG32(scratch);
1058                 if (tmp == 0xDEADBEEF)
1059                         break;
1060                 udelay(1);
1061         }
1062
1063         if (i >= adev->usec_timeout)
1064                 r = -ETIMEDOUT;
1065
1066 error_free_scratch:
1067         amdgpu_gfx_scratch_free(adev, scratch);
1068         return r;
1069 }
1070
1071 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1072 {
1073         struct amdgpu_device *adev = ring->adev;
1074         struct amdgpu_ib ib;
1075         struct dma_fence *f = NULL;
1076
1077         unsigned index;
1078         uint64_t gpu_addr;
1079         uint32_t tmp;
1080         long r;
1081
1082         r = amdgpu_device_wb_get(adev, &index);
1083         if (r)
1084                 return r;
1085
1086         gpu_addr = adev->wb.gpu_addr + (index * 4);
1087         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1088         memset(&ib, 0, sizeof(ib));
1089         r = amdgpu_ib_get(adev, NULL, 16,
1090                                         AMDGPU_IB_POOL_DIRECT, &ib);
1091         if (r)
1092                 goto err1;
1093
1094         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1095         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1096         ib.ptr[2] = lower_32_bits(gpu_addr);
1097         ib.ptr[3] = upper_32_bits(gpu_addr);
1098         ib.ptr[4] = 0xDEADBEEF;
1099         ib.length_dw = 5;
1100
1101         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1102         if (r)
1103                 goto err2;
1104
1105         r = dma_fence_wait_timeout(f, false, timeout);
1106         if (r == 0) {
1107                 r = -ETIMEDOUT;
1108                 goto err2;
1109         } else if (r < 0) {
1110                 goto err2;
1111         }
1112
1113         tmp = adev->wb.wb[index];
1114         if (tmp == 0xDEADBEEF)
1115                 r = 0;
1116         else
1117                 r = -EINVAL;
1118
1119 err2:
1120         amdgpu_ib_free(adev, &ib, NULL);
1121         dma_fence_put(f);
1122 err1:
1123         amdgpu_device_wb_free(adev, index);
1124         return r;
1125 }
1126
1127
1128 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1129 {
1130         release_firmware(adev->gfx.pfp_fw);
1131         adev->gfx.pfp_fw = NULL;
1132         release_firmware(adev->gfx.me_fw);
1133         adev->gfx.me_fw = NULL;
1134         release_firmware(adev->gfx.ce_fw);
1135         adev->gfx.ce_fw = NULL;
1136         release_firmware(adev->gfx.rlc_fw);
1137         adev->gfx.rlc_fw = NULL;
1138         release_firmware(adev->gfx.mec_fw);
1139         adev->gfx.mec_fw = NULL;
1140         release_firmware(adev->gfx.mec2_fw);
1141         adev->gfx.mec2_fw = NULL;
1142
1143         kfree(adev->gfx.rlc.register_list_format);
1144 }
1145
1146 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
1147 {
1148         const struct rlc_firmware_header_v2_1 *rlc_hdr;
1149
1150         rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1151         adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
1152         adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
1153         adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
1154         adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
1155         adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
1156         adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
1157         adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
1158         adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
1159         adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
1160         adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
1161         adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
1162         adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
1163         adev->gfx.rlc.reg_list_format_direct_reg_list_length =
1164                         le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
1165 }
1166
1167 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1168 {
1169         adev->gfx.me_fw_write_wait = false;
1170         adev->gfx.mec_fw_write_wait = false;
1171
1172         if ((adev->asic_type != CHIP_ARCTURUS) &&
1173             ((adev->gfx.mec_fw_version < 0x000001a5) ||
1174             (adev->gfx.mec_feature_version < 46) ||
1175             (adev->gfx.pfp_fw_version < 0x000000b7) ||
1176             (adev->gfx.pfp_feature_version < 46)))
1177                 DRM_WARN_ONCE("CP firmware version too old, please update!");
1178
1179         switch (adev->asic_type) {
1180         case CHIP_VEGA10:
1181                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1182                     (adev->gfx.me_feature_version >= 42) &&
1183                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1184                     (adev->gfx.pfp_feature_version >= 42))
1185                         adev->gfx.me_fw_write_wait = true;
1186
1187                 if ((adev->gfx.mec_fw_version >=  0x00000193) &&
1188                     (adev->gfx.mec_feature_version >= 42))
1189                         adev->gfx.mec_fw_write_wait = true;
1190                 break;
1191         case CHIP_VEGA12:
1192                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1193                     (adev->gfx.me_feature_version >= 44) &&
1194                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1195                     (adev->gfx.pfp_feature_version >= 44))
1196                         adev->gfx.me_fw_write_wait = true;
1197
1198                 if ((adev->gfx.mec_fw_version >=  0x00000196) &&
1199                     (adev->gfx.mec_feature_version >= 44))
1200                         adev->gfx.mec_fw_write_wait = true;
1201                 break;
1202         case CHIP_VEGA20:
1203                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1204                     (adev->gfx.me_feature_version >= 44) &&
1205                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1206                     (adev->gfx.pfp_feature_version >= 44))
1207                         adev->gfx.me_fw_write_wait = true;
1208
1209                 if ((adev->gfx.mec_fw_version >=  0x00000197) &&
1210                     (adev->gfx.mec_feature_version >= 44))
1211                         adev->gfx.mec_fw_write_wait = true;
1212                 break;
1213         case CHIP_RAVEN:
1214                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1215                     (adev->gfx.me_feature_version >= 42) &&
1216                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1217                     (adev->gfx.pfp_feature_version >= 42))
1218                         adev->gfx.me_fw_write_wait = true;
1219
1220                 if ((adev->gfx.mec_fw_version >=  0x00000192) &&
1221                     (adev->gfx.mec_feature_version >= 42))
1222                         adev->gfx.mec_fw_write_wait = true;
1223                 break;
1224         default:
1225                 adev->gfx.me_fw_write_wait = true;
1226                 adev->gfx.mec_fw_write_wait = true;
1227                 break;
1228         }
1229 }
1230
1231 struct amdgpu_gfxoff_quirk {
1232         u16 chip_vendor;
1233         u16 chip_device;
1234         u16 subsys_vendor;
1235         u16 subsys_device;
1236         u8 revision;
1237 };
1238
1239 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1240         /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1241         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1242         /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1243         { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1244         /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1245         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1246         { 0, 0, 0, 0, 0 },
1247 };
1248
1249 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1250 {
1251         const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1252
1253         while (p && p->chip_device != 0) {
1254                 if (pdev->vendor == p->chip_vendor &&
1255                     pdev->device == p->chip_device &&
1256                     pdev->subsystem_vendor == p->subsys_vendor &&
1257                     pdev->subsystem_device == p->subsys_device &&
1258                     pdev->revision == p->revision) {
1259                         return true;
1260                 }
1261                 ++p;
1262         }
1263         return false;
1264 }
1265
1266 static bool is_raven_kicker(struct amdgpu_device *adev)
1267 {
1268         if (adev->pm.fw_version >= 0x41e2b)
1269                 return true;
1270         else
1271                 return false;
1272 }
1273
1274 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1275 {
1276         if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1277                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1278
1279         switch (adev->asic_type) {
1280         case CHIP_VEGA10:
1281         case CHIP_VEGA12:
1282         case CHIP_VEGA20:
1283                 break;
1284         case CHIP_RAVEN:
1285                 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1286                       (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1287                     ((!is_raven_kicker(adev) &&
1288                       adev->gfx.rlc_fw_version < 531) ||
1289                      (adev->gfx.rlc_feature_version < 1) ||
1290                      !adev->gfx.rlc.is_rlc_v2_1))
1291                         adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1292
1293                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1294                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1295                                 AMD_PG_SUPPORT_CP |
1296                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1297                 break;
1298         case CHIP_RENOIR:
1299                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1300                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1301                                 AMD_PG_SUPPORT_CP |
1302                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1303                 break;
1304         default:
1305                 break;
1306         }
1307 }
1308
1309 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1310                                           const char *chip_name)
1311 {
1312         char fw_name[30];
1313         int err;
1314         struct amdgpu_firmware_info *info = NULL;
1315         const struct common_firmware_header *header = NULL;
1316         const struct gfx_firmware_header_v1_0 *cp_hdr;
1317
1318         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1319         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1320         if (err)
1321                 goto out;
1322         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1323         if (err)
1324                 goto out;
1325         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1326         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1327         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1328
1329         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1330         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1331         if (err)
1332                 goto out;
1333         err = amdgpu_ucode_validate(adev->gfx.me_fw);
1334         if (err)
1335                 goto out;
1336         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1337         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1338         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1339
1340         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1341         err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1342         if (err)
1343                 goto out;
1344         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1345         if (err)
1346                 goto out;
1347         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1348         adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1349         adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1350
1351         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1352                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1353                 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1354                 info->fw = adev->gfx.pfp_fw;
1355                 header = (const struct common_firmware_header *)info->fw->data;
1356                 adev->firmware.fw_size +=
1357                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1358
1359                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1360                 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1361                 info->fw = adev->gfx.me_fw;
1362                 header = (const struct common_firmware_header *)info->fw->data;
1363                 adev->firmware.fw_size +=
1364                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1365
1366                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1367                 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1368                 info->fw = adev->gfx.ce_fw;
1369                 header = (const struct common_firmware_header *)info->fw->data;
1370                 adev->firmware.fw_size +=
1371                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1372         }
1373
1374 out:
1375         if (err) {
1376                 dev_err(adev->dev,
1377                         "gfx9: Failed to load firmware \"%s\"\n",
1378                         fw_name);
1379                 release_firmware(adev->gfx.pfp_fw);
1380                 adev->gfx.pfp_fw = NULL;
1381                 release_firmware(adev->gfx.me_fw);
1382                 adev->gfx.me_fw = NULL;
1383                 release_firmware(adev->gfx.ce_fw);
1384                 adev->gfx.ce_fw = NULL;
1385         }
1386         return err;
1387 }
1388
1389 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1390                                           const char *chip_name)
1391 {
1392         char fw_name[30];
1393         int err;
1394         struct amdgpu_firmware_info *info = NULL;
1395         const struct common_firmware_header *header = NULL;
1396         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1397         unsigned int *tmp = NULL;
1398         unsigned int i = 0;
1399         uint16_t version_major;
1400         uint16_t version_minor;
1401         uint32_t smu_version;
1402
1403         /*
1404          * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1405          * instead of picasso_rlc.bin.
1406          * Judgment method:
1407          * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1408          *          or revision >= 0xD8 && revision <= 0xDF
1409          * otherwise is PCO FP5
1410          */
1411         if (!strcmp(chip_name, "picasso") &&
1412                 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1413                 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1414                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1415         else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1416                 (smu_version >= 0x41e2b))
1417                 /**
1418                 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1419                 */
1420                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1421         else
1422                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1423         err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1424         if (err)
1425                 goto out;
1426         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1427         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1428
1429         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1430         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1431         if (version_major == 2 && version_minor == 1)
1432                 adev->gfx.rlc.is_rlc_v2_1 = true;
1433
1434         adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1435         adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1436         adev->gfx.rlc.save_and_restore_offset =
1437                         le32_to_cpu(rlc_hdr->save_and_restore_offset);
1438         adev->gfx.rlc.clear_state_descriptor_offset =
1439                         le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1440         adev->gfx.rlc.avail_scratch_ram_locations =
1441                         le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1442         adev->gfx.rlc.reg_restore_list_size =
1443                         le32_to_cpu(rlc_hdr->reg_restore_list_size);
1444         adev->gfx.rlc.reg_list_format_start =
1445                         le32_to_cpu(rlc_hdr->reg_list_format_start);
1446         adev->gfx.rlc.reg_list_format_separate_start =
1447                         le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1448         adev->gfx.rlc.starting_offsets_start =
1449                         le32_to_cpu(rlc_hdr->starting_offsets_start);
1450         adev->gfx.rlc.reg_list_format_size_bytes =
1451                         le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1452         adev->gfx.rlc.reg_list_size_bytes =
1453                         le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1454         adev->gfx.rlc.register_list_format =
1455                         kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1456                                 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1457         if (!adev->gfx.rlc.register_list_format) {
1458                 err = -ENOMEM;
1459                 goto out;
1460         }
1461
1462         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1463                         le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1464         for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1465                 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1466
1467         adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1468
1469         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1470                         le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1471         for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1472                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1473
1474         if (adev->gfx.rlc.is_rlc_v2_1)
1475                 gfx_v9_0_init_rlc_ext_microcode(adev);
1476
1477         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1478                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1479                 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1480                 info->fw = adev->gfx.rlc_fw;
1481                 header = (const struct common_firmware_header *)info->fw->data;
1482                 adev->firmware.fw_size +=
1483                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1484
1485                 if (adev->gfx.rlc.is_rlc_v2_1 &&
1486                     adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
1487                     adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
1488                     adev->gfx.rlc.save_restore_list_srm_size_bytes) {
1489                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
1490                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
1491                         info->fw = adev->gfx.rlc_fw;
1492                         adev->firmware.fw_size +=
1493                                 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
1494
1495                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
1496                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
1497                         info->fw = adev->gfx.rlc_fw;
1498                         adev->firmware.fw_size +=
1499                                 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
1500
1501                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
1502                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
1503                         info->fw = adev->gfx.rlc_fw;
1504                         adev->firmware.fw_size +=
1505                                 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
1506                 }
1507         }
1508
1509 out:
1510         if (err) {
1511                 dev_err(adev->dev,
1512                         "gfx9: Failed to load firmware \"%s\"\n",
1513                         fw_name);
1514                 release_firmware(adev->gfx.rlc_fw);
1515                 adev->gfx.rlc_fw = NULL;
1516         }
1517         return err;
1518 }
1519
1520 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1521                                           const char *chip_name)
1522 {
1523         char fw_name[30];
1524         int err;
1525         struct amdgpu_firmware_info *info = NULL;
1526         const struct common_firmware_header *header = NULL;
1527         const struct gfx_firmware_header_v1_0 *cp_hdr;
1528
1529         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1530         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1531         if (err)
1532                 goto out;
1533         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1534         if (err)
1535                 goto out;
1536         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1537         adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1538         adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1539
1540
1541         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1542         err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1543         if (!err) {
1544                 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1545                 if (err)
1546                         goto out;
1547                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1548                 adev->gfx.mec2_fw->data;
1549                 adev->gfx.mec2_fw_version =
1550                 le32_to_cpu(cp_hdr->header.ucode_version);
1551                 adev->gfx.mec2_feature_version =
1552                 le32_to_cpu(cp_hdr->ucode_feature_version);
1553         } else {
1554                 err = 0;
1555                 adev->gfx.mec2_fw = NULL;
1556         }
1557
1558         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1559                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1560                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1561                 info->fw = adev->gfx.mec_fw;
1562                 header = (const struct common_firmware_header *)info->fw->data;
1563                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1564                 adev->firmware.fw_size +=
1565                         ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1566
1567                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
1568                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
1569                 info->fw = adev->gfx.mec_fw;
1570                 adev->firmware.fw_size +=
1571                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1572
1573                 if (adev->gfx.mec2_fw) {
1574                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1575                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1576                         info->fw = adev->gfx.mec2_fw;
1577                         header = (const struct common_firmware_header *)info->fw->data;
1578                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1579                         adev->firmware.fw_size +=
1580                                 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1581
1582                         /* TODO: Determine if MEC2 JT FW loading can be removed
1583                                  for all GFX V9 asic and above */
1584                         if (adev->asic_type != CHIP_ARCTURUS &&
1585                             adev->asic_type != CHIP_RENOIR) {
1586                                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
1587                                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
1588                                 info->fw = adev->gfx.mec2_fw;
1589                                 adev->firmware.fw_size +=
1590                                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
1591                                         PAGE_SIZE);
1592                         }
1593                 }
1594         }
1595
1596 out:
1597         gfx_v9_0_check_if_need_gfxoff(adev);
1598         gfx_v9_0_check_fw_write_wait(adev);
1599         if (err) {
1600                 dev_err(adev->dev,
1601                         "gfx9: Failed to load firmware \"%s\"\n",
1602                         fw_name);
1603                 release_firmware(adev->gfx.mec_fw);
1604                 adev->gfx.mec_fw = NULL;
1605                 release_firmware(adev->gfx.mec2_fw);
1606                 adev->gfx.mec2_fw = NULL;
1607         }
1608         return err;
1609 }
1610
1611 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1612 {
1613         const char *chip_name;
1614         int r;
1615
1616         DRM_DEBUG("\n");
1617
1618         switch (adev->asic_type) {
1619         case CHIP_VEGA10:
1620                 chip_name = "vega10";
1621                 break;
1622         case CHIP_VEGA12:
1623                 chip_name = "vega12";
1624                 break;
1625         case CHIP_VEGA20:
1626                 chip_name = "vega20";
1627                 break;
1628         case CHIP_RAVEN:
1629                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1630                         chip_name = "raven2";
1631                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1632                         chip_name = "picasso";
1633                 else
1634                         chip_name = "raven";
1635                 break;
1636         case CHIP_ARCTURUS:
1637                 chip_name = "arcturus";
1638                 break;
1639         case CHIP_RENOIR:
1640                 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1641                         chip_name = "renoir";
1642                 else
1643                         chip_name = "green_sardine";
1644                 break;
1645         default:
1646                 BUG();
1647         }
1648
1649         /* No CPG in Arcturus */
1650         if (adev->gfx.num_gfx_rings) {
1651                 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name);
1652                 if (r)
1653                         return r;
1654         }
1655
1656         r = gfx_v9_0_init_rlc_microcode(adev, chip_name);
1657         if (r)
1658                 return r;
1659
1660         r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name);
1661         if (r)
1662                 return r;
1663
1664         return r;
1665 }
1666
1667 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1668 {
1669         u32 count = 0;
1670         const struct cs_section_def *sect = NULL;
1671         const struct cs_extent_def *ext = NULL;
1672
1673         /* begin clear state */
1674         count += 2;
1675         /* context control state */
1676         count += 3;
1677
1678         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1679                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1680                         if (sect->id == SECT_CONTEXT)
1681                                 count += 2 + ext->reg_count;
1682                         else
1683                                 return 0;
1684                 }
1685         }
1686
1687         /* end clear state */
1688         count += 2;
1689         /* clear state */
1690         count += 2;
1691
1692         return count;
1693 }
1694
1695 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1696                                     volatile u32 *buffer)
1697 {
1698         u32 count = 0, i;
1699         const struct cs_section_def *sect = NULL;
1700         const struct cs_extent_def *ext = NULL;
1701
1702         if (adev->gfx.rlc.cs_data == NULL)
1703                 return;
1704         if (buffer == NULL)
1705                 return;
1706
1707         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1708         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1709
1710         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1711         buffer[count++] = cpu_to_le32(0x80000000);
1712         buffer[count++] = cpu_to_le32(0x80000000);
1713
1714         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1715                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1716                         if (sect->id == SECT_CONTEXT) {
1717                                 buffer[count++] =
1718                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1719                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1720                                                 PACKET3_SET_CONTEXT_REG_START);
1721                                 for (i = 0; i < ext->reg_count; i++)
1722                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1723                         } else {
1724                                 return;
1725                         }
1726                 }
1727         }
1728
1729         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1730         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1731
1732         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1733         buffer[count++] = cpu_to_le32(0);
1734 }
1735
1736 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1737 {
1738         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1739         uint32_t pg_always_on_cu_num = 2;
1740         uint32_t always_on_cu_num;
1741         uint32_t i, j, k;
1742         uint32_t mask, cu_bitmap, counter;
1743
1744         if (adev->flags & AMD_IS_APU)
1745                 always_on_cu_num = 4;
1746         else if (adev->asic_type == CHIP_VEGA12)
1747                 always_on_cu_num = 8;
1748         else
1749                 always_on_cu_num = 12;
1750
1751         mutex_lock(&adev->grbm_idx_mutex);
1752         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1753                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1754                         mask = 1;
1755                         cu_bitmap = 0;
1756                         counter = 0;
1757                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1758
1759                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1760                                 if (cu_info->bitmap[i][j] & mask) {
1761                                         if (counter == pg_always_on_cu_num)
1762                                                 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1763                                         if (counter < always_on_cu_num)
1764                                                 cu_bitmap |= mask;
1765                                         else
1766                                                 break;
1767                                         counter++;
1768                                 }
1769                                 mask <<= 1;
1770                         }
1771
1772                         WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1773                         cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1774                 }
1775         }
1776         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1777         mutex_unlock(&adev->grbm_idx_mutex);
1778 }
1779
1780 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1781 {
1782         uint32_t data;
1783
1784         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1785         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1786         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1787         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1788         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1789
1790         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1791         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1792
1793         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1794         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1795
1796         mutex_lock(&adev->grbm_idx_mutex);
1797         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1798         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1799         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1800
1801         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1802         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1803         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1804         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1805         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1806
1807         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1808         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1809         data &= 0x0000FFFF;
1810         data |= 0x00C00000;
1811         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1812
1813         /*
1814          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1815          * programmed in gfx_v9_0_init_always_on_cu_mask()
1816          */
1817
1818         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1819          * but used for RLC_LB_CNTL configuration */
1820         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1821         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1822         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1823         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1824         mutex_unlock(&adev->grbm_idx_mutex);
1825
1826         gfx_v9_0_init_always_on_cu_mask(adev);
1827 }
1828
1829 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1830 {
1831         uint32_t data;
1832
1833         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1834         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1835         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1836         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1837         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1838
1839         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1840         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1841
1842         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1843         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1844
1845         mutex_lock(&adev->grbm_idx_mutex);
1846         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1847         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1848         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1849
1850         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1851         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1852         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1853         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1854         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1855
1856         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1857         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1858         data &= 0x0000FFFF;
1859         data |= 0x00C00000;
1860         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1861
1862         /*
1863          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1864          * programmed in gfx_v9_0_init_always_on_cu_mask()
1865          */
1866
1867         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1868          * but used for RLC_LB_CNTL configuration */
1869         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1870         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1871         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1872         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1873         mutex_unlock(&adev->grbm_idx_mutex);
1874
1875         gfx_v9_0_init_always_on_cu_mask(adev);
1876 }
1877
1878 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1879 {
1880         WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1881 }
1882
1883 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1884 {
1885         return 5;
1886 }
1887
1888 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1889 {
1890         const struct cs_section_def *cs_data;
1891         int r;
1892
1893         adev->gfx.rlc.cs_data = gfx9_cs_data;
1894
1895         cs_data = adev->gfx.rlc.cs_data;
1896
1897         if (cs_data) {
1898                 /* init clear state block */
1899                 r = amdgpu_gfx_rlc_init_csb(adev);
1900                 if (r)
1901                         return r;
1902         }
1903
1904         if (adev->flags & AMD_IS_APU) {
1905                 /* TODO: double check the cp_table_size for RV */
1906                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1907                 r = amdgpu_gfx_rlc_init_cpt(adev);
1908                 if (r)
1909                         return r;
1910         }
1911
1912         switch (adev->asic_type) {
1913         case CHIP_RAVEN:
1914                 gfx_v9_0_init_lbpw(adev);
1915                 break;
1916         case CHIP_VEGA20:
1917                 gfx_v9_4_init_lbpw(adev);
1918                 break;
1919         default:
1920                 break;
1921         }
1922
1923         /* init spm vmid with 0xf */
1924         if (adev->gfx.rlc.funcs->update_spm_vmid)
1925                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1926
1927         return 0;
1928 }
1929
1930 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1931 {
1932         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1933         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1934 }
1935
1936 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1937 {
1938         int r;
1939         u32 *hpd;
1940         const __le32 *fw_data;
1941         unsigned fw_size;
1942         u32 *fw;
1943         size_t mec_hpd_size;
1944
1945         const struct gfx_firmware_header_v1_0 *mec_hdr;
1946
1947         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1948
1949         /* take ownership of the relevant compute queues */
1950         amdgpu_gfx_compute_queue_acquire(adev);
1951         mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1952         if (mec_hpd_size) {
1953                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1954                                               AMDGPU_GEM_DOMAIN_VRAM,
1955                                               &adev->gfx.mec.hpd_eop_obj,
1956                                               &adev->gfx.mec.hpd_eop_gpu_addr,
1957                                               (void **)&hpd);
1958                 if (r) {
1959                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1960                         gfx_v9_0_mec_fini(adev);
1961                         return r;
1962                 }
1963
1964                 memset(hpd, 0, mec_hpd_size);
1965
1966                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1967                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1968         }
1969
1970         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1971
1972         fw_data = (const __le32 *)
1973                 (adev->gfx.mec_fw->data +
1974                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1975         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1976
1977         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1978                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1979                                       &adev->gfx.mec.mec_fw_obj,
1980                                       &adev->gfx.mec.mec_fw_gpu_addr,
1981                                       (void **)&fw);
1982         if (r) {
1983                 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1984                 gfx_v9_0_mec_fini(adev);
1985                 return r;
1986         }
1987
1988         memcpy(fw, fw_data, fw_size);
1989
1990         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1991         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1992
1993         return 0;
1994 }
1995
1996 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1997 {
1998         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1999                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2000                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2001                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
2002                 (SQ_IND_INDEX__FORCE_READ_MASK));
2003         return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2004 }
2005
2006 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
2007                            uint32_t wave, uint32_t thread,
2008                            uint32_t regno, uint32_t num, uint32_t *out)
2009 {
2010         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2011                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2012                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2013                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
2014                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
2015                 (SQ_IND_INDEX__FORCE_READ_MASK) |
2016                 (SQ_IND_INDEX__AUTO_INCR_MASK));
2017         while (num--)
2018                 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2019 }
2020
2021 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
2022 {
2023         /* type 1 wave data */
2024         dst[(*no_fields)++] = 1;
2025         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
2026         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
2027         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
2028         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
2029         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
2030         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
2031         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
2032         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
2033         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
2034         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
2035         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
2036         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
2037         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
2038         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
2039 }
2040
2041 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
2042                                      uint32_t wave, uint32_t start,
2043                                      uint32_t size, uint32_t *dst)
2044 {
2045         wave_read_regs(
2046                 adev, simd, wave, 0,
2047                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
2048 }
2049
2050 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
2051                                      uint32_t wave, uint32_t thread,
2052                                      uint32_t start, uint32_t size,
2053                                      uint32_t *dst)
2054 {
2055         wave_read_regs(
2056                 adev, simd, wave, thread,
2057                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
2058 }
2059
2060 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
2061                                   u32 me, u32 pipe, u32 q, u32 vm)
2062 {
2063         soc15_grbm_select(adev, me, pipe, q, vm);
2064 }
2065
2066 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2067         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2068         .select_se_sh = &gfx_v9_0_select_se_sh,
2069         .read_wave_data = &gfx_v9_0_read_wave_data,
2070         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2071         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2072         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2073         .ras_error_inject = &gfx_v9_0_ras_error_inject,
2074         .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2075         .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2076 };
2077
2078 static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = {
2079         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2080         .select_se_sh = &gfx_v9_0_select_se_sh,
2081         .read_wave_data = &gfx_v9_0_read_wave_data,
2082         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2083         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2084         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2085         .ras_error_inject = &gfx_v9_4_ras_error_inject,
2086         .query_ras_error_count = &gfx_v9_4_query_ras_error_count,
2087         .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count,
2088         .query_ras_error_status = &gfx_v9_4_query_ras_error_status,
2089 };
2090
2091 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2092 {
2093         u32 gb_addr_config;
2094         int err;
2095
2096         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
2097
2098         switch (adev->asic_type) {
2099         case CHIP_VEGA10:
2100                 adev->gfx.config.max_hw_contexts = 8;
2101                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2102                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2103                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2104                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2105                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2106                 break;
2107         case CHIP_VEGA12:
2108                 adev->gfx.config.max_hw_contexts = 8;
2109                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2110                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2111                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2112                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2113                 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2114                 DRM_INFO("fix gfx.config for vega12\n");
2115                 break;
2116         case CHIP_VEGA20:
2117                 adev->gfx.config.max_hw_contexts = 8;
2118                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2119                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2120                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2121                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2122                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2123                 gb_addr_config &= ~0xf3e777ff;
2124                 gb_addr_config |= 0x22014042;
2125                 /* check vbios table if gpu info is not available */
2126                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2127                 if (err)
2128                         return err;
2129                 break;
2130         case CHIP_RAVEN:
2131                 adev->gfx.config.max_hw_contexts = 8;
2132                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2133                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2134                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2135                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2136                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2137                         gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2138                 else
2139                         gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2140                 break;
2141         case CHIP_ARCTURUS:
2142                 adev->gfx.funcs = &gfx_v9_4_gfx_funcs;
2143                 adev->gfx.config.max_hw_contexts = 8;
2144                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2145                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2146                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2147                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2148                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2149                 gb_addr_config &= ~0xf3e777ff;
2150                 gb_addr_config |= 0x22014042;
2151                 break;
2152         case CHIP_RENOIR:
2153                 adev->gfx.config.max_hw_contexts = 8;
2154                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2155                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2156                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2157                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2158                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2159                 gb_addr_config &= ~0xf3e777ff;
2160                 gb_addr_config |= 0x22010042;
2161                 break;
2162         default:
2163                 BUG();
2164                 break;
2165         }
2166
2167         adev->gfx.config.gb_addr_config = gb_addr_config;
2168
2169         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2170                         REG_GET_FIELD(
2171                                         adev->gfx.config.gb_addr_config,
2172                                         GB_ADDR_CONFIG,
2173                                         NUM_PIPES);
2174
2175         adev->gfx.config.max_tile_pipes =
2176                 adev->gfx.config.gb_addr_config_fields.num_pipes;
2177
2178         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2179                         REG_GET_FIELD(
2180                                         adev->gfx.config.gb_addr_config,
2181                                         GB_ADDR_CONFIG,
2182                                         NUM_BANKS);
2183         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2184                         REG_GET_FIELD(
2185                                         adev->gfx.config.gb_addr_config,
2186                                         GB_ADDR_CONFIG,
2187                                         MAX_COMPRESSED_FRAGS);
2188         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2189                         REG_GET_FIELD(
2190                                         adev->gfx.config.gb_addr_config,
2191                                         GB_ADDR_CONFIG,
2192                                         NUM_RB_PER_SE);
2193         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2194                         REG_GET_FIELD(
2195                                         adev->gfx.config.gb_addr_config,
2196                                         GB_ADDR_CONFIG,
2197                                         NUM_SHADER_ENGINES);
2198         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2199                         REG_GET_FIELD(
2200                                         adev->gfx.config.gb_addr_config,
2201                                         GB_ADDR_CONFIG,
2202                                         PIPE_INTERLEAVE_SIZE));
2203
2204         return 0;
2205 }
2206
2207 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2208                                       int mec, int pipe, int queue)
2209 {
2210         unsigned irq_type;
2211         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2212         unsigned int hw_prio;
2213
2214         ring = &adev->gfx.compute_ring[ring_id];
2215
2216         /* mec0 is me1 */
2217         ring->me = mec + 1;
2218         ring->pipe = pipe;
2219         ring->queue = queue;
2220
2221         ring->ring_obj = NULL;
2222         ring->use_doorbell = true;
2223         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2224         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2225                                 + (ring_id * GFX9_MEC_HPD_SIZE);
2226         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2227
2228         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2229                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2230                 + ring->pipe;
2231         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2232                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
2233         /* type-2 packets are deprecated on MEC, use type-3 instead */
2234         return amdgpu_ring_init(adev, ring, 1024,
2235                                 &adev->gfx.eop_irq, irq_type, hw_prio);
2236 }
2237
2238 static int gfx_v9_0_sw_init(void *handle)
2239 {
2240         int i, j, k, r, ring_id;
2241         struct amdgpu_ring *ring;
2242         struct amdgpu_kiq *kiq;
2243         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2244
2245         switch (adev->asic_type) {
2246         case CHIP_VEGA10:
2247         case CHIP_VEGA12:
2248         case CHIP_VEGA20:
2249         case CHIP_RAVEN:
2250         case CHIP_ARCTURUS:
2251         case CHIP_RENOIR:
2252                 adev->gfx.mec.num_mec = 2;
2253                 break;
2254         default:
2255                 adev->gfx.mec.num_mec = 1;
2256                 break;
2257         }
2258
2259         adev->gfx.mec.num_pipe_per_mec = 4;
2260         adev->gfx.mec.num_queue_per_pipe = 8;
2261
2262         /* EOP Event */
2263         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2264         if (r)
2265                 return r;
2266
2267         /* Privileged reg */
2268         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2269                               &adev->gfx.priv_reg_irq);
2270         if (r)
2271                 return r;
2272
2273         /* Privileged inst */
2274         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2275                               &adev->gfx.priv_inst_irq);
2276         if (r)
2277                 return r;
2278
2279         /* ECC error */
2280         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2281                               &adev->gfx.cp_ecc_error_irq);
2282         if (r)
2283                 return r;
2284
2285         /* FUE error */
2286         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2287                               &adev->gfx.cp_ecc_error_irq);
2288         if (r)
2289                 return r;
2290
2291         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2292
2293         gfx_v9_0_scratch_init(adev);
2294
2295         r = gfx_v9_0_init_microcode(adev);
2296         if (r) {
2297                 DRM_ERROR("Failed to load gfx firmware!\n");
2298                 return r;
2299         }
2300
2301         r = adev->gfx.rlc.funcs->init(adev);
2302         if (r) {
2303                 DRM_ERROR("Failed to init rlc BOs!\n");
2304                 return r;
2305         }
2306
2307         r = gfx_v9_0_mec_init(adev);
2308         if (r) {
2309                 DRM_ERROR("Failed to init MEC BOs!\n");
2310                 return r;
2311         }
2312
2313         /* set up the gfx ring */
2314         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2315                 ring = &adev->gfx.gfx_ring[i];
2316                 ring->ring_obj = NULL;
2317                 if (!i)
2318                         sprintf(ring->name, "gfx");
2319                 else
2320                         sprintf(ring->name, "gfx_%d", i);
2321                 ring->use_doorbell = true;
2322                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2323                 r = amdgpu_ring_init(adev, ring, 1024,
2324                                      &adev->gfx.eop_irq,
2325                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2326                                      AMDGPU_RING_PRIO_DEFAULT);
2327                 if (r)
2328                         return r;
2329         }
2330
2331         /* set up the compute queues - allocate horizontally across pipes */
2332         ring_id = 0;
2333         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2334                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2335                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2336                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2337                                         continue;
2338
2339                                 r = gfx_v9_0_compute_ring_init(adev,
2340                                                                ring_id,
2341                                                                i, k, j);
2342                                 if (r)
2343                                         return r;
2344
2345                                 ring_id++;
2346                         }
2347                 }
2348         }
2349
2350         r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2351         if (r) {
2352                 DRM_ERROR("Failed to init KIQ BOs!\n");
2353                 return r;
2354         }
2355
2356         kiq = &adev->gfx.kiq;
2357         r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2358         if (r)
2359                 return r;
2360
2361         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2362         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2363         if (r)
2364                 return r;
2365
2366         adev->gfx.ce_ram_size = 0x8000;
2367
2368         r = gfx_v9_0_gpu_early_init(adev);
2369         if (r)
2370                 return r;
2371
2372         return 0;
2373 }
2374
2375
2376 static int gfx_v9_0_sw_fini(void *handle)
2377 {
2378         int i;
2379         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2380
2381         amdgpu_gfx_ras_fini(adev);
2382
2383         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2384                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2385         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2386                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2387
2388         amdgpu_gfx_mqd_sw_fini(adev);
2389         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2390         amdgpu_gfx_kiq_fini(adev);
2391
2392         gfx_v9_0_mec_fini(adev);
2393         amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
2394         if (adev->flags & AMD_IS_APU) {
2395                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2396                                 &adev->gfx.rlc.cp_table_gpu_addr,
2397                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2398         }
2399         gfx_v9_0_free_microcode(adev);
2400
2401         return 0;
2402 }
2403
2404
2405 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2406 {
2407         /* TODO */
2408 }
2409
2410 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2411                            u32 instance)
2412 {
2413         u32 data;
2414
2415         if (instance == 0xffffffff)
2416                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2417         else
2418                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2419
2420         if (se_num == 0xffffffff)
2421                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2422         else
2423                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2424
2425         if (sh_num == 0xffffffff)
2426                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2427         else
2428                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2429
2430         WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2431 }
2432
2433 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2434 {
2435         u32 data, mask;
2436
2437         data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2438         data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2439
2440         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2441         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2442
2443         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2444                                          adev->gfx.config.max_sh_per_se);
2445
2446         return (~data) & mask;
2447 }
2448
2449 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2450 {
2451         int i, j;
2452         u32 data;
2453         u32 active_rbs = 0;
2454         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2455                                         adev->gfx.config.max_sh_per_se;
2456
2457         mutex_lock(&adev->grbm_idx_mutex);
2458         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2459                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2460                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2461                         data = gfx_v9_0_get_rb_active_bitmap(adev);
2462                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2463                                                rb_bitmap_width_per_sh);
2464                 }
2465         }
2466         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2467         mutex_unlock(&adev->grbm_idx_mutex);
2468
2469         adev->gfx.config.backend_enable_mask = active_rbs;
2470         adev->gfx.config.num_rbs = hweight32(active_rbs);
2471 }
2472
2473 #define DEFAULT_SH_MEM_BASES    (0x6000)
2474 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2475 {
2476         int i;
2477         uint32_t sh_mem_config;
2478         uint32_t sh_mem_bases;
2479
2480         /*
2481          * Configure apertures:
2482          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
2483          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
2484          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
2485          */
2486         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2487
2488         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2489                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2490                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2491
2492         mutex_lock(&adev->srbm_mutex);
2493         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2494                 soc15_grbm_select(adev, 0, 0, 0, i);
2495                 /* CP and shaders */
2496                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2497                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2498         }
2499         soc15_grbm_select(adev, 0, 0, 0, 0);
2500         mutex_unlock(&adev->srbm_mutex);
2501
2502         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2503            acccess. These should be enabled by FW for target VMIDs. */
2504         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2505                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2506                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2507                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2508                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2509         }
2510 }
2511
2512 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2513 {
2514         int vmid;
2515
2516         /*
2517          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2518          * access. Compute VMIDs should be enabled by FW for target VMIDs,
2519          * the driver can enable them for graphics. VMID0 should maintain
2520          * access so that HWS firmware can save/restore entries.
2521          */
2522         for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2523                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2524                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2525                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2526                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2527         }
2528 }
2529
2530 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2531 {
2532         uint32_t tmp;
2533
2534         switch (adev->asic_type) {
2535         case CHIP_ARCTURUS:
2536                 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2537                 tmp = REG_SET_FIELD(tmp, SQ_CONFIG,
2538                                         DISABLE_BARRIER_WAITCNT, 1);
2539                 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2540                 break;
2541         default:
2542                 break;
2543         }
2544 }
2545
2546 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2547 {
2548         u32 tmp;
2549         int i;
2550
2551         WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2552
2553         gfx_v9_0_tiling_mode_table_init(adev);
2554
2555         gfx_v9_0_setup_rb(adev);
2556         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2557         adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2558
2559         /* XXX SH_MEM regs */
2560         /* where to put LDS, scratch, GPUVM in FSA64 space */
2561         mutex_lock(&adev->srbm_mutex);
2562         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2563                 soc15_grbm_select(adev, 0, 0, 0, i);
2564                 /* CP and shaders */
2565                 if (i == 0) {
2566                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2567                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2568                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2569                                             !!adev->gmc.noretry);
2570                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2571                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2572                 } else {
2573                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2574                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2575                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2576                                             !!adev->gmc.noretry);
2577                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2578                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2579                                 (adev->gmc.private_aperture_start >> 48));
2580                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2581                                 (adev->gmc.shared_aperture_start >> 48));
2582                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2583                 }
2584         }
2585         soc15_grbm_select(adev, 0, 0, 0, 0);
2586
2587         mutex_unlock(&adev->srbm_mutex);
2588
2589         gfx_v9_0_init_compute_vmid(adev);
2590         gfx_v9_0_init_gds_vmid(adev);
2591         gfx_v9_0_init_sq_config(adev);
2592 }
2593
2594 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2595 {
2596         u32 i, j, k;
2597         u32 mask;
2598
2599         mutex_lock(&adev->grbm_idx_mutex);
2600         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2601                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2602                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2603                         for (k = 0; k < adev->usec_timeout; k++) {
2604                                 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2605                                         break;
2606                                 udelay(1);
2607                         }
2608                         if (k == adev->usec_timeout) {
2609                                 gfx_v9_0_select_se_sh(adev, 0xffffffff,
2610                                                       0xffffffff, 0xffffffff);
2611                                 mutex_unlock(&adev->grbm_idx_mutex);
2612                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2613                                          i, j);
2614                                 return;
2615                         }
2616                 }
2617         }
2618         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2619         mutex_unlock(&adev->grbm_idx_mutex);
2620
2621         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2622                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2623                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2624                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2625         for (k = 0; k < adev->usec_timeout; k++) {
2626                 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2627                         break;
2628                 udelay(1);
2629         }
2630 }
2631
2632 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2633                                                bool enable)
2634 {
2635         u32 tmp;
2636
2637         /* don't toggle interrupts that are only applicable
2638          * to me0 pipe0 on AISCs that have me0 removed */
2639         if (!adev->gfx.num_gfx_rings)
2640                 return;
2641
2642         tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2643
2644         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2645         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2646         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2647         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2648
2649         WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2650 }
2651
2652 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2653 {
2654         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2655         /* csib */
2656         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2657                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
2658         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2659                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2660         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2661                         adev->gfx.rlc.clear_state_size);
2662 }
2663
2664 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2665                                 int indirect_offset,
2666                                 int list_size,
2667                                 int *unique_indirect_regs,
2668                                 int unique_indirect_reg_count,
2669                                 int *indirect_start_offsets,
2670                                 int *indirect_start_offsets_count,
2671                                 int max_start_offsets_count)
2672 {
2673         int idx;
2674
2675         for (; indirect_offset < list_size; indirect_offset++) {
2676                 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2677                 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2678                 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2679
2680                 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2681                         indirect_offset += 2;
2682
2683                         /* look for the matching indice */
2684                         for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2685                                 if (unique_indirect_regs[idx] ==
2686                                         register_list_format[indirect_offset] ||
2687                                         !unique_indirect_regs[idx])
2688                                         break;
2689                         }
2690
2691                         BUG_ON(idx >= unique_indirect_reg_count);
2692
2693                         if (!unique_indirect_regs[idx])
2694                                 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2695
2696                         indirect_offset++;
2697                 }
2698         }
2699 }
2700
2701 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2702 {
2703         int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2704         int unique_indirect_reg_count = 0;
2705
2706         int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2707         int indirect_start_offsets_count = 0;
2708
2709         int list_size = 0;
2710         int i = 0, j = 0;
2711         u32 tmp = 0;
2712
2713         u32 *register_list_format =
2714                 kmemdup(adev->gfx.rlc.register_list_format,
2715                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2716         if (!register_list_format)
2717                 return -ENOMEM;
2718
2719         /* setup unique_indirect_regs array and indirect_start_offsets array */
2720         unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2721         gfx_v9_1_parse_ind_reg_list(register_list_format,
2722                                     adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2723                                     adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2724                                     unique_indirect_regs,
2725                                     unique_indirect_reg_count,
2726                                     indirect_start_offsets,
2727                                     &indirect_start_offsets_count,
2728                                     ARRAY_SIZE(indirect_start_offsets));
2729
2730         /* enable auto inc in case it is disabled */
2731         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2732         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2733         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2734
2735         /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2736         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2737                 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2738         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2739                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2740                         adev->gfx.rlc.register_restore[i]);
2741
2742         /* load indirect register */
2743         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2744                 adev->gfx.rlc.reg_list_format_start);
2745
2746         /* direct register portion */
2747         for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2748                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2749                         register_list_format[i]);
2750
2751         /* indirect register portion */
2752         while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2753                 if (register_list_format[i] == 0xFFFFFFFF) {
2754                         WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2755                         continue;
2756                 }
2757
2758                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2759                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2760
2761                 for (j = 0; j < unique_indirect_reg_count; j++) {
2762                         if (register_list_format[i] == unique_indirect_regs[j]) {
2763                                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2764                                 break;
2765                         }
2766                 }
2767
2768                 BUG_ON(j >= unique_indirect_reg_count);
2769
2770                 i++;
2771         }
2772
2773         /* set save/restore list size */
2774         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2775         list_size = list_size >> 1;
2776         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2777                 adev->gfx.rlc.reg_restore_list_size);
2778         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2779
2780         /* write the starting offsets to RLC scratch ram */
2781         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2782                 adev->gfx.rlc.starting_offsets_start);
2783         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2784                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2785                        indirect_start_offsets[i]);
2786
2787         /* load unique indirect regs*/
2788         for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2789                 if (unique_indirect_regs[i] != 0) {
2790                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2791                                + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2792                                unique_indirect_regs[i] & 0x3FFFF);
2793
2794                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2795                                + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2796                                unique_indirect_regs[i] >> 20);
2797                 }
2798         }
2799
2800         kfree(register_list_format);
2801         return 0;
2802 }
2803
2804 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2805 {
2806         WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2807 }
2808
2809 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2810                                              bool enable)
2811 {
2812         uint32_t data = 0;
2813         uint32_t default_data = 0;
2814
2815         default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2816         if (enable) {
2817                 /* enable GFXIP control over CGPG */
2818                 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2819                 if(default_data != data)
2820                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2821
2822                 /* update status */
2823                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2824                 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2825                 if(default_data != data)
2826                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2827         } else {
2828                 /* restore GFXIP control over GCPG */
2829                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2830                 if(default_data != data)
2831                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2832         }
2833 }
2834
2835 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2836 {
2837         uint32_t data = 0;
2838
2839         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2840                               AMD_PG_SUPPORT_GFX_SMG |
2841                               AMD_PG_SUPPORT_GFX_DMG)) {
2842                 /* init IDLE_POLL_COUNT = 60 */
2843                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2844                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2845                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2846                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2847
2848                 /* init RLC PG Delay */
2849                 data = 0;
2850                 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2851                 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2852                 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2853                 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2854                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2855
2856                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2857                 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2858                 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2859                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2860
2861                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2862                 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2863                 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2864                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2865
2866                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2867                 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2868
2869                 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2870                 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2871                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2872                 if (adev->asic_type != CHIP_RENOIR)
2873                         pwr_10_0_gfxip_control_over_cgpg(adev, true);
2874         }
2875 }
2876
2877 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2878                                                 bool enable)
2879 {
2880         uint32_t data = 0;
2881         uint32_t default_data = 0;
2882
2883         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2884         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2885                              SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2886                              enable ? 1 : 0);
2887         if (default_data != data)
2888                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2889 }
2890
2891 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2892                                                 bool enable)
2893 {
2894         uint32_t data = 0;
2895         uint32_t default_data = 0;
2896
2897         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2898         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2899                              SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2900                              enable ? 1 : 0);
2901         if(default_data != data)
2902                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2903 }
2904
2905 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2906                                         bool enable)
2907 {
2908         uint32_t data = 0;
2909         uint32_t default_data = 0;
2910
2911         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2912         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2913                              CP_PG_DISABLE,
2914                              enable ? 0 : 1);
2915         if(default_data != data)
2916                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2917 }
2918
2919 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2920                                                 bool enable)
2921 {
2922         uint32_t data, default_data;
2923
2924         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2925         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2926                              GFX_POWER_GATING_ENABLE,
2927                              enable ? 1 : 0);
2928         if(default_data != data)
2929                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2930 }
2931
2932 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2933                                                 bool enable)
2934 {
2935         uint32_t data, default_data;
2936
2937         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2938         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2939                              GFX_PIPELINE_PG_ENABLE,
2940                              enable ? 1 : 0);
2941         if(default_data != data)
2942                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2943
2944         if (!enable)
2945                 /* read any GFX register to wake up GFX */
2946                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2947 }
2948
2949 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
2950                                                        bool enable)
2951 {
2952         uint32_t data, default_data;
2953
2954         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2955         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2956                              STATIC_PER_CU_PG_ENABLE,
2957                              enable ? 1 : 0);
2958         if(default_data != data)
2959                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2960 }
2961
2962 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
2963                                                 bool enable)
2964 {
2965         uint32_t data, default_data;
2966
2967         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2968         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2969                              DYN_PER_CU_PG_ENABLE,
2970                              enable ? 1 : 0);
2971         if(default_data != data)
2972                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2973 }
2974
2975 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
2976 {
2977         gfx_v9_0_init_csb(adev);
2978
2979         /*
2980          * Rlc save restore list is workable since v2_1.
2981          * And it's needed by gfxoff feature.
2982          */
2983         if (adev->gfx.rlc.is_rlc_v2_1) {
2984                 if (adev->asic_type == CHIP_VEGA12 ||
2985                     (adev->apu_flags & AMD_APU_IS_RAVEN2))
2986                         gfx_v9_1_init_rlc_save_restore_list(adev);
2987                 gfx_v9_0_enable_save_restore_machine(adev);
2988         }
2989
2990         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2991                               AMD_PG_SUPPORT_GFX_SMG |
2992                               AMD_PG_SUPPORT_GFX_DMG |
2993                               AMD_PG_SUPPORT_CP |
2994                               AMD_PG_SUPPORT_GDS |
2995                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
2996                 WREG32(mmRLC_JUMP_TABLE_RESTORE,
2997                        adev->gfx.rlc.cp_table_gpu_addr >> 8);
2998                 gfx_v9_0_init_gfx_power_gating(adev);
2999         }
3000 }
3001
3002 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3003 {
3004         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3005         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3006         gfx_v9_0_wait_for_rlc_serdes(adev);
3007 }
3008
3009 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3010 {
3011         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3012         udelay(50);
3013         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3014         udelay(50);
3015 }
3016
3017 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3018 {
3019 #ifdef AMDGPU_RLC_DEBUG_RETRY
3020         u32 rlc_ucode_ver;
3021 #endif
3022
3023         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3024         udelay(50);
3025
3026         /* carrizo do enable cp interrupt after cp inited */
3027         if (!(adev->flags & AMD_IS_APU)) {
3028                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3029                 udelay(50);
3030         }
3031
3032 #ifdef AMDGPU_RLC_DEBUG_RETRY
3033         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3034         rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3035         if(rlc_ucode_ver == 0x108) {
3036                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3037                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3038                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3039                  * default is 0x9C4 to create a 100us interval */
3040                 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3041                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3042                  * to disable the page fault retry interrupts, default is
3043                  * 0x100 (256) */
3044                 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3045         }
3046 #endif
3047 }
3048
3049 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3050 {
3051         const struct rlc_firmware_header_v2_0 *hdr;
3052         const __le32 *fw_data;
3053         unsigned i, fw_size;
3054
3055         if (!adev->gfx.rlc_fw)
3056                 return -EINVAL;
3057
3058         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3059         amdgpu_ucode_print_rlc_hdr(&hdr->header);
3060
3061         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3062                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3063         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3064
3065         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3066                         RLCG_UCODE_LOADING_START_ADDRESS);
3067         for (i = 0; i < fw_size; i++)
3068                 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3069         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3070
3071         return 0;
3072 }
3073
3074 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3075 {
3076         int r;
3077
3078         if (amdgpu_sriov_vf(adev)) {
3079                 gfx_v9_0_init_csb(adev);
3080                 return 0;
3081         }
3082
3083         adev->gfx.rlc.funcs->stop(adev);
3084
3085         /* disable CG */
3086         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3087
3088         gfx_v9_0_init_pg(adev);
3089
3090         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3091                 /* legacy rlc firmware loading */
3092                 r = gfx_v9_0_rlc_load_microcode(adev);
3093                 if (r)
3094                         return r;
3095         }
3096
3097         switch (adev->asic_type) {
3098         case CHIP_RAVEN:
3099                 if (amdgpu_lbpw == 0)
3100                         gfx_v9_0_enable_lbpw(adev, false);
3101                 else
3102                         gfx_v9_0_enable_lbpw(adev, true);
3103                 break;
3104         case CHIP_VEGA20:
3105                 if (amdgpu_lbpw > 0)
3106                         gfx_v9_0_enable_lbpw(adev, true);
3107                 else
3108                         gfx_v9_0_enable_lbpw(adev, false);
3109                 break;
3110         default:
3111                 break;
3112         }
3113
3114         adev->gfx.rlc.funcs->start(adev);
3115
3116         return 0;
3117 }
3118
3119 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3120 {
3121         u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3122
3123         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3124         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3125         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3126         WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3127         udelay(50);
3128 }
3129
3130 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3131 {
3132         const struct gfx_firmware_header_v1_0 *pfp_hdr;
3133         const struct gfx_firmware_header_v1_0 *ce_hdr;
3134         const struct gfx_firmware_header_v1_0 *me_hdr;
3135         const __le32 *fw_data;
3136         unsigned i, fw_size;
3137
3138         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3139                 return -EINVAL;
3140
3141         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3142                 adev->gfx.pfp_fw->data;
3143         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3144                 adev->gfx.ce_fw->data;
3145         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3146                 adev->gfx.me_fw->data;
3147
3148         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3149         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3150         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3151
3152         gfx_v9_0_cp_gfx_enable(adev, false);
3153
3154         /* PFP */
3155         fw_data = (const __le32 *)
3156                 (adev->gfx.pfp_fw->data +
3157                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3158         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3159         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3160         for (i = 0; i < fw_size; i++)
3161                 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3162         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3163
3164         /* CE */
3165         fw_data = (const __le32 *)
3166                 (adev->gfx.ce_fw->data +
3167                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3168         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3169         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3170         for (i = 0; i < fw_size; i++)
3171                 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3172         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3173
3174         /* ME */
3175         fw_data = (const __le32 *)
3176                 (adev->gfx.me_fw->data +
3177                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3178         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3179         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3180         for (i = 0; i < fw_size; i++)
3181                 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3182         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3183
3184         return 0;
3185 }
3186
3187 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3188 {
3189         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3190         const struct cs_section_def *sect = NULL;
3191         const struct cs_extent_def *ext = NULL;
3192         int r, i, tmp;
3193
3194         /* init the CP */
3195         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3196         WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3197
3198         gfx_v9_0_cp_gfx_enable(adev, true);
3199
3200         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3201         if (r) {
3202                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3203                 return r;
3204         }
3205
3206         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3207         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3208
3209         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3210         amdgpu_ring_write(ring, 0x80000000);
3211         amdgpu_ring_write(ring, 0x80000000);
3212
3213         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3214                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3215                         if (sect->id == SECT_CONTEXT) {
3216                                 amdgpu_ring_write(ring,
3217                                        PACKET3(PACKET3_SET_CONTEXT_REG,
3218                                                ext->reg_count));
3219                                 amdgpu_ring_write(ring,
3220                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3221                                 for (i = 0; i < ext->reg_count; i++)
3222                                         amdgpu_ring_write(ring, ext->extent[i]);
3223                         }
3224                 }
3225         }
3226
3227         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3228         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3229
3230         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3231         amdgpu_ring_write(ring, 0);
3232
3233         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3234         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3235         amdgpu_ring_write(ring, 0x8000);
3236         amdgpu_ring_write(ring, 0x8000);
3237
3238         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3239         tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3240                 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3241         amdgpu_ring_write(ring, tmp);
3242         amdgpu_ring_write(ring, 0);
3243
3244         amdgpu_ring_commit(ring);
3245
3246         return 0;
3247 }
3248
3249 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3250 {
3251         struct amdgpu_ring *ring;
3252         u32 tmp;
3253         u32 rb_bufsz;
3254         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3255
3256         /* Set the write pointer delay */
3257         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3258
3259         /* set the RB to use vmid 0 */
3260         WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3261
3262         /* Set ring buffer size */
3263         ring = &adev->gfx.gfx_ring[0];
3264         rb_bufsz = order_base_2(ring->ring_size / 8);
3265         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3266         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3267 #ifdef __BIG_ENDIAN
3268         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3269 #endif
3270         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3271
3272         /* Initialize the ring buffer's write pointers */
3273         ring->wptr = 0;
3274         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3275         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3276
3277         /* set the wb address wether it's enabled or not */
3278         rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3279         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3280         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3281
3282         wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3283         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3284         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3285
3286         mdelay(1);
3287         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3288
3289         rb_addr = ring->gpu_addr >> 8;
3290         WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3291         WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3292
3293         tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3294         if (ring->use_doorbell) {
3295                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3296                                     DOORBELL_OFFSET, ring->doorbell_index);
3297                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3298                                     DOORBELL_EN, 1);
3299         } else {
3300                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3301         }
3302         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3303
3304         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3305                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
3306         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3307
3308         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3309                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3310
3311
3312         /* start the ring */
3313         gfx_v9_0_cp_gfx_start(adev);
3314         ring->sched.ready = true;
3315
3316         return 0;
3317 }
3318
3319 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3320 {
3321         if (enable) {
3322                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3323         } else {
3324                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3325                         (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3326                 adev->gfx.kiq.ring.sched.ready = false;
3327         }
3328         udelay(50);
3329 }
3330
3331 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3332 {
3333         const struct gfx_firmware_header_v1_0 *mec_hdr;
3334         const __le32 *fw_data;
3335         unsigned i;
3336         u32 tmp;
3337
3338         if (!adev->gfx.mec_fw)
3339                 return -EINVAL;
3340
3341         gfx_v9_0_cp_compute_enable(adev, false);
3342
3343         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3344         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3345
3346         fw_data = (const __le32 *)
3347                 (adev->gfx.mec_fw->data +
3348                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3349         tmp = 0;
3350         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3351         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3352         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3353
3354         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3355                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3356         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3357                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3358
3359         /* MEC1 */
3360         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3361                          mec_hdr->jt_offset);
3362         for (i = 0; i < mec_hdr->jt_size; i++)
3363                 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3364                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3365
3366         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3367                         adev->gfx.mec_fw_version);
3368         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3369
3370         return 0;
3371 }
3372
3373 /* KIQ functions */
3374 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3375 {
3376         uint32_t tmp;
3377         struct amdgpu_device *adev = ring->adev;
3378
3379         /* tell RLC which is KIQ queue */
3380         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3381         tmp &= 0xffffff00;
3382         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3383         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3384         tmp |= 0x80;
3385         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3386 }
3387
3388 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3389 {
3390         struct amdgpu_device *adev = ring->adev;
3391
3392         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3393                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3394                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3395                         mqd->cp_hqd_queue_priority =
3396                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3397                 }
3398         }
3399 }
3400
3401 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3402 {
3403         struct amdgpu_device *adev = ring->adev;
3404         struct v9_mqd *mqd = ring->mqd_ptr;
3405         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3406         uint32_t tmp;
3407
3408         mqd->header = 0xC0310800;
3409         mqd->compute_pipelinestat_enable = 0x00000001;
3410         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3411         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3412         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3413         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3414         mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3415         mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3416         mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3417         mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3418         mqd->compute_misc_reserved = 0x00000003;
3419
3420         mqd->dynamic_cu_mask_addr_lo =
3421                 lower_32_bits(ring->mqd_gpu_addr
3422                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3423         mqd->dynamic_cu_mask_addr_hi =
3424                 upper_32_bits(ring->mqd_gpu_addr
3425                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3426
3427         eop_base_addr = ring->eop_gpu_addr >> 8;
3428         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3429         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3430
3431         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3432         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3433         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3434                         (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3435
3436         mqd->cp_hqd_eop_control = tmp;
3437
3438         /* enable doorbell? */
3439         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3440
3441         if (ring->use_doorbell) {
3442                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3443                                     DOORBELL_OFFSET, ring->doorbell_index);
3444                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3445                                     DOORBELL_EN, 1);
3446                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3447                                     DOORBELL_SOURCE, 0);
3448                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3449                                     DOORBELL_HIT, 0);
3450         } else {
3451                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3452                                          DOORBELL_EN, 0);
3453         }
3454
3455         mqd->cp_hqd_pq_doorbell_control = tmp;
3456
3457         /* disable the queue if it's active */
3458         ring->wptr = 0;
3459         mqd->cp_hqd_dequeue_request = 0;
3460         mqd->cp_hqd_pq_rptr = 0;
3461         mqd->cp_hqd_pq_wptr_lo = 0;
3462         mqd->cp_hqd_pq_wptr_hi = 0;
3463
3464         /* set the pointer to the MQD */
3465         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3466         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3467
3468         /* set MQD vmid to 0 */
3469         tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3470         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3471         mqd->cp_mqd_control = tmp;
3472
3473         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3474         hqd_gpu_addr = ring->gpu_addr >> 8;
3475         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3476         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3477
3478         /* set up the HQD, this is similar to CP_RB0_CNTL */
3479         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3480         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3481                             (order_base_2(ring->ring_size / 4) - 1));
3482         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3483                         ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3484 #ifdef __BIG_ENDIAN
3485         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3486 #endif
3487         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3488         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3489         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3490         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3491         mqd->cp_hqd_pq_control = tmp;
3492
3493         /* set the wb address whether it's enabled or not */
3494         wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3495         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3496         mqd->cp_hqd_pq_rptr_report_addr_hi =
3497                 upper_32_bits(wb_gpu_addr) & 0xffff;
3498
3499         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3500         wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3501         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3502         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3503
3504         tmp = 0;
3505         /* enable the doorbell if requested */
3506         if (ring->use_doorbell) {
3507                 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3508                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3509                                 DOORBELL_OFFSET, ring->doorbell_index);
3510
3511                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3512                                          DOORBELL_EN, 1);
3513                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3514                                          DOORBELL_SOURCE, 0);
3515                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3516                                          DOORBELL_HIT, 0);
3517         }
3518
3519         mqd->cp_hqd_pq_doorbell_control = tmp;
3520
3521         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3522         ring->wptr = 0;
3523         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3524
3525         /* set the vmid for the queue */
3526         mqd->cp_hqd_vmid = 0;
3527
3528         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3529         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3530         mqd->cp_hqd_persistent_state = tmp;
3531
3532         /* set MIN_IB_AVAIL_SIZE */
3533         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3534         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3535         mqd->cp_hqd_ib_control = tmp;
3536
3537         /* set static priority for a queue/ring */
3538         gfx_v9_0_mqd_set_priority(ring, mqd);
3539         mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
3540
3541         /* map_queues packet doesn't need activate the queue,
3542          * so only kiq need set this field.
3543          */
3544         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3545                 mqd->cp_hqd_active = 1;
3546
3547         return 0;
3548 }
3549
3550 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3551 {
3552         struct amdgpu_device *adev = ring->adev;
3553         struct v9_mqd *mqd = ring->mqd_ptr;
3554         int j;
3555
3556         /* disable wptr polling */
3557         WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3558
3559         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3560                mqd->cp_hqd_eop_base_addr_lo);
3561         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3562                mqd->cp_hqd_eop_base_addr_hi);
3563
3564         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3565         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3566                mqd->cp_hqd_eop_control);
3567
3568         /* enable doorbell? */
3569         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3570                mqd->cp_hqd_pq_doorbell_control);
3571
3572         /* disable the queue if it's active */
3573         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3574                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3575                 for (j = 0; j < adev->usec_timeout; j++) {
3576                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3577                                 break;
3578                         udelay(1);
3579                 }
3580                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3581                        mqd->cp_hqd_dequeue_request);
3582                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3583                        mqd->cp_hqd_pq_rptr);
3584                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3585                        mqd->cp_hqd_pq_wptr_lo);
3586                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3587                        mqd->cp_hqd_pq_wptr_hi);
3588         }
3589
3590         /* set the pointer to the MQD */
3591         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3592                mqd->cp_mqd_base_addr_lo);
3593         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3594                mqd->cp_mqd_base_addr_hi);
3595
3596         /* set MQD vmid to 0 */
3597         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3598                mqd->cp_mqd_control);
3599
3600         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3601         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3602                mqd->cp_hqd_pq_base_lo);
3603         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3604                mqd->cp_hqd_pq_base_hi);
3605
3606         /* set up the HQD, this is similar to CP_RB0_CNTL */
3607         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3608                mqd->cp_hqd_pq_control);
3609
3610         /* set the wb address whether it's enabled or not */
3611         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3612                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3613         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3614                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3615
3616         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3617         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3618                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3619         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3620                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3621
3622         /* enable the doorbell if requested */
3623         if (ring->use_doorbell) {
3624                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3625                                         (adev->doorbell_index.kiq * 2) << 2);
3626                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3627                                         (adev->doorbell_index.userqueue_end * 2) << 2);
3628         }
3629
3630         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3631                mqd->cp_hqd_pq_doorbell_control);
3632
3633         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3634         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3635                mqd->cp_hqd_pq_wptr_lo);
3636         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3637                mqd->cp_hqd_pq_wptr_hi);
3638
3639         /* set the vmid for the queue */
3640         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3641
3642         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3643                mqd->cp_hqd_persistent_state);
3644
3645         /* activate the queue */
3646         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3647                mqd->cp_hqd_active);
3648
3649         if (ring->use_doorbell)
3650                 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3651
3652         return 0;
3653 }
3654
3655 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3656 {
3657         struct amdgpu_device *adev = ring->adev;
3658         int j;
3659
3660         /* disable the queue if it's active */
3661         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3662
3663                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3664
3665                 for (j = 0; j < adev->usec_timeout; j++) {
3666                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3667                                 break;
3668                         udelay(1);
3669                 }
3670
3671                 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3672                         DRM_DEBUG("KIQ dequeue request failed.\n");
3673
3674                         /* Manual disable if dequeue request times out */
3675                         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3676                 }
3677
3678                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3679                       0);
3680         }
3681
3682         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3683         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3684         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3685         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3686         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3687         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3688         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3689         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3690
3691         return 0;
3692 }
3693
3694 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3695 {
3696         struct amdgpu_device *adev = ring->adev;
3697         struct v9_mqd *mqd = ring->mqd_ptr;
3698         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3699
3700         gfx_v9_0_kiq_setting(ring);
3701
3702         if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
3703                 /* reset MQD to a clean status */
3704                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3705                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3706
3707                 /* reset ring buffer */
3708                 ring->wptr = 0;
3709                 amdgpu_ring_clear_ring(ring);
3710
3711                 mutex_lock(&adev->srbm_mutex);
3712                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3713                 gfx_v9_0_kiq_init_register(ring);
3714                 soc15_grbm_select(adev, 0, 0, 0, 0);
3715                 mutex_unlock(&adev->srbm_mutex);
3716         } else {
3717                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3718                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3719                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3720                 mutex_lock(&adev->srbm_mutex);
3721                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3722                 gfx_v9_0_mqd_init(ring);
3723                 gfx_v9_0_kiq_init_register(ring);
3724                 soc15_grbm_select(adev, 0, 0, 0, 0);
3725                 mutex_unlock(&adev->srbm_mutex);
3726
3727                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3728                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3729         }
3730
3731         return 0;
3732 }
3733
3734 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3735 {
3736         struct amdgpu_device *adev = ring->adev;
3737         struct v9_mqd *mqd = ring->mqd_ptr;
3738         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3739
3740         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3741                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3742                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3743                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3744                 mutex_lock(&adev->srbm_mutex);
3745                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3746                 gfx_v9_0_mqd_init(ring);
3747                 soc15_grbm_select(adev, 0, 0, 0, 0);
3748                 mutex_unlock(&adev->srbm_mutex);
3749
3750                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3751                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3752         } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
3753                 /* reset MQD to a clean status */
3754                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3755                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3756
3757                 /* reset ring buffer */
3758                 ring->wptr = 0;
3759                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
3760                 amdgpu_ring_clear_ring(ring);
3761         } else {
3762                 amdgpu_ring_clear_ring(ring);
3763         }
3764
3765         return 0;
3766 }
3767
3768 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3769 {
3770         struct amdgpu_ring *ring;
3771         int r;
3772
3773         ring = &adev->gfx.kiq.ring;
3774
3775         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3776         if (unlikely(r != 0))
3777                 return r;
3778
3779         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3780         if (unlikely(r != 0))
3781                 return r;
3782
3783         gfx_v9_0_kiq_init_queue(ring);
3784         amdgpu_bo_kunmap(ring->mqd_obj);
3785         ring->mqd_ptr = NULL;
3786         amdgpu_bo_unreserve(ring->mqd_obj);
3787         ring->sched.ready = true;
3788         return 0;
3789 }
3790
3791 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3792 {
3793         struct amdgpu_ring *ring = NULL;
3794         int r = 0, i;
3795
3796         gfx_v9_0_cp_compute_enable(adev, true);
3797
3798         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3799                 ring = &adev->gfx.compute_ring[i];
3800
3801                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3802                 if (unlikely(r != 0))
3803                         goto done;
3804                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3805                 if (!r) {
3806                         r = gfx_v9_0_kcq_init_queue(ring);
3807                         amdgpu_bo_kunmap(ring->mqd_obj);
3808                         ring->mqd_ptr = NULL;
3809                 }
3810                 amdgpu_bo_unreserve(ring->mqd_obj);
3811                 if (r)
3812                         goto done;
3813         }
3814
3815         r = amdgpu_gfx_enable_kcq(adev);
3816 done:
3817         return r;
3818 }
3819
3820 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3821 {
3822         int r, i;
3823         struct amdgpu_ring *ring;
3824
3825         if (!(adev->flags & AMD_IS_APU))
3826                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3827
3828         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3829                 if (adev->gfx.num_gfx_rings) {
3830                         /* legacy firmware loading */
3831                         r = gfx_v9_0_cp_gfx_load_microcode(adev);
3832                         if (r)
3833                                 return r;
3834                 }
3835
3836                 r = gfx_v9_0_cp_compute_load_microcode(adev);
3837                 if (r)
3838                         return r;
3839         }
3840
3841         r = gfx_v9_0_kiq_resume(adev);
3842         if (r)
3843                 return r;
3844
3845         if (adev->gfx.num_gfx_rings) {
3846                 r = gfx_v9_0_cp_gfx_resume(adev);
3847                 if (r)
3848                         return r;
3849         }
3850
3851         r = gfx_v9_0_kcq_resume(adev);
3852         if (r)
3853                 return r;
3854
3855         if (adev->gfx.num_gfx_rings) {
3856                 ring = &adev->gfx.gfx_ring[0];
3857                 r = amdgpu_ring_test_helper(ring);
3858                 if (r)
3859                         return r;
3860         }
3861
3862         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3863                 ring = &adev->gfx.compute_ring[i];
3864                 amdgpu_ring_test_helper(ring);
3865         }
3866
3867         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3868
3869         return 0;
3870 }
3871
3872 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3873 {
3874         u32 tmp;
3875
3876         if (adev->asic_type != CHIP_ARCTURUS)
3877                 return;
3878
3879         tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3880         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3881                                 adev->df.hash_status.hash_64k);
3882         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3883                                 adev->df.hash_status.hash_2m);
3884         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3885                                 adev->df.hash_status.hash_1g);
3886         WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3887 }
3888
3889 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3890 {
3891         if (adev->gfx.num_gfx_rings)
3892                 gfx_v9_0_cp_gfx_enable(adev, enable);
3893         gfx_v9_0_cp_compute_enable(adev, enable);
3894 }
3895
3896 static int gfx_v9_0_hw_init(void *handle)
3897 {
3898         int r;
3899         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3900
3901         if (!amdgpu_sriov_vf(adev))
3902                 gfx_v9_0_init_golden_registers(adev);
3903
3904         gfx_v9_0_constants_init(adev);
3905
3906         gfx_v9_0_init_tcp_config(adev);
3907
3908         r = adev->gfx.rlc.funcs->resume(adev);
3909         if (r)
3910                 return r;
3911
3912         r = gfx_v9_0_cp_resume(adev);
3913         if (r)
3914                 return r;
3915
3916         return r;
3917 }
3918
3919 static int gfx_v9_0_hw_fini(void *handle)
3920 {
3921         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3922
3923         amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
3924         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3925         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3926
3927         /* DF freeze and kcq disable will fail */
3928         if (!amdgpu_ras_intr_triggered())
3929                 /* disable KCQ to avoid CPC touch memory not valid anymore */
3930                 amdgpu_gfx_disable_kcq(adev);
3931
3932         if (amdgpu_sriov_vf(adev)) {
3933                 gfx_v9_0_cp_gfx_enable(adev, false);
3934                 /* must disable polling for SRIOV when hw finished, otherwise
3935                  * CPC engine may still keep fetching WB address which is already
3936                  * invalid after sw finished and trigger DMAR reading error in
3937                  * hypervisor side.
3938                  */
3939                 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3940                 return 0;
3941         }
3942
3943         /* Use deinitialize sequence from CAIL when unbinding device from driver,
3944          * otherwise KIQ is hanging when binding back
3945          */
3946         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3947                 mutex_lock(&adev->srbm_mutex);
3948                 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
3949                                 adev->gfx.kiq.ring.pipe,
3950                                 adev->gfx.kiq.ring.queue, 0);
3951                 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
3952                 soc15_grbm_select(adev, 0, 0, 0, 0);
3953                 mutex_unlock(&adev->srbm_mutex);
3954         }
3955
3956         gfx_v9_0_cp_enable(adev, false);
3957         adev->gfx.rlc.funcs->stop(adev);
3958
3959         return 0;
3960 }
3961
3962 static int gfx_v9_0_suspend(void *handle)
3963 {
3964         return gfx_v9_0_hw_fini(handle);
3965 }
3966
3967 static int gfx_v9_0_resume(void *handle)
3968 {
3969         return gfx_v9_0_hw_init(handle);
3970 }
3971
3972 static bool gfx_v9_0_is_idle(void *handle)
3973 {
3974         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3975
3976         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
3977                                 GRBM_STATUS, GUI_ACTIVE))
3978                 return false;
3979         else
3980                 return true;
3981 }
3982
3983 static int gfx_v9_0_wait_for_idle(void *handle)
3984 {
3985         unsigned i;
3986         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3987
3988         for (i = 0; i < adev->usec_timeout; i++) {
3989                 if (gfx_v9_0_is_idle(handle))
3990                         return 0;
3991                 udelay(1);
3992         }
3993         return -ETIMEDOUT;
3994 }
3995
3996 static int gfx_v9_0_soft_reset(void *handle)
3997 {
3998         u32 grbm_soft_reset = 0;
3999         u32 tmp;
4000         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4001
4002         /* GRBM_STATUS */
4003         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4004         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4005                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4006                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4007                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4008                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4009                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4010                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4011                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4012                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4013                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4014         }
4015
4016         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4017                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4018                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4019         }
4020
4021         /* GRBM_STATUS2 */
4022         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4023         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4024                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4025                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4026
4027
4028         if (grbm_soft_reset) {
4029                 /* stop the rlc */
4030                 adev->gfx.rlc.funcs->stop(adev);
4031
4032                 if (adev->gfx.num_gfx_rings)
4033                         /* Disable GFX parsing/prefetching */
4034                         gfx_v9_0_cp_gfx_enable(adev, false);
4035
4036                 /* Disable MEC parsing/prefetching */
4037                 gfx_v9_0_cp_compute_enable(adev, false);
4038
4039                 if (grbm_soft_reset) {
4040                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4041                         tmp |= grbm_soft_reset;
4042                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4043                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4044                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4045
4046                         udelay(50);
4047
4048                         tmp &= ~grbm_soft_reset;
4049                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4050                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4051                 }
4052
4053                 /* Wait a little for things to settle down */
4054                 udelay(50);
4055         }
4056         return 0;
4057 }
4058
4059 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4060 {
4061         signed long r, cnt = 0;
4062         unsigned long flags;
4063         uint32_t seq, reg_val_offs = 0;
4064         uint64_t value = 0;
4065         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4066         struct amdgpu_ring *ring = &kiq->ring;
4067
4068         BUG_ON(!ring->funcs->emit_rreg);
4069
4070         spin_lock_irqsave(&kiq->ring_lock, flags);
4071         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
4072                 pr_err("critical bug! too many kiq readers\n");
4073                 goto failed_unlock;
4074         }
4075         amdgpu_ring_alloc(ring, 32);
4076         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4077         amdgpu_ring_write(ring, 9 |     /* src: register*/
4078                                 (5 << 8) |      /* dst: memory */
4079                                 (1 << 16) |     /* count sel */
4080                                 (1 << 20));     /* write confirm */
4081         amdgpu_ring_write(ring, 0);
4082         amdgpu_ring_write(ring, 0);
4083         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4084                                 reg_val_offs * 4));
4085         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4086                                 reg_val_offs * 4));
4087         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4088         if (r)
4089                 goto failed_undo;
4090
4091         amdgpu_ring_commit(ring);
4092         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4093
4094         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4095
4096         /* don't wait anymore for gpu reset case because this way may
4097          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4098          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4099          * never return if we keep waiting in virt_kiq_rreg, which cause
4100          * gpu_recover() hang there.
4101          *
4102          * also don't wait anymore for IRQ context
4103          * */
4104         if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
4105                 goto failed_kiq_read;
4106
4107         might_sleep();
4108         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4109                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4110                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4111         }
4112
4113         if (cnt > MAX_KIQ_REG_TRY)
4114                 goto failed_kiq_read;
4115
4116         mb();
4117         value = (uint64_t)adev->wb.wb[reg_val_offs] |
4118                 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4119         amdgpu_device_wb_free(adev, reg_val_offs);
4120         return value;
4121
4122 failed_undo:
4123         amdgpu_ring_undo(ring);
4124 failed_unlock:
4125         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4126 failed_kiq_read:
4127         if (reg_val_offs)
4128                 amdgpu_device_wb_free(adev, reg_val_offs);
4129         pr_err("failed to read gpu clock\n");
4130         return ~0;
4131 }
4132
4133 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4134 {
4135         uint64_t clock;
4136
4137         amdgpu_gfx_off_ctrl(adev, false);
4138         mutex_lock(&adev->gfx.gpu_clock_mutex);
4139         if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
4140                 clock = gfx_v9_0_kiq_read_clock(adev);
4141         } else {
4142                 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4143                 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4144                         ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4145         }
4146         mutex_unlock(&adev->gfx.gpu_clock_mutex);
4147         amdgpu_gfx_off_ctrl(adev, true);
4148         return clock;
4149 }
4150
4151 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4152                                           uint32_t vmid,
4153                                           uint32_t gds_base, uint32_t gds_size,
4154                                           uint32_t gws_base, uint32_t gws_size,
4155                                           uint32_t oa_base, uint32_t oa_size)
4156 {
4157         struct amdgpu_device *adev = ring->adev;
4158
4159         /* GDS Base */
4160         gfx_v9_0_write_data_to_reg(ring, 0, false,
4161                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4162                                    gds_base);
4163
4164         /* GDS Size */
4165         gfx_v9_0_write_data_to_reg(ring, 0, false,
4166                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4167                                    gds_size);
4168
4169         /* GWS */
4170         gfx_v9_0_write_data_to_reg(ring, 0, false,
4171                                    SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4172                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4173
4174         /* OA */
4175         gfx_v9_0_write_data_to_reg(ring, 0, false,
4176                                    SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4177                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
4178 }
4179
4180 static const u32 vgpr_init_compute_shader[] =
4181 {
4182         0xb07c0000, 0xbe8000ff,
4183         0x000000f8, 0xbf110800,
4184         0x7e000280, 0x7e020280,
4185         0x7e040280, 0x7e060280,
4186         0x7e080280, 0x7e0a0280,
4187         0x7e0c0280, 0x7e0e0280,
4188         0x80808800, 0xbe803200,
4189         0xbf84fff5, 0xbf9c0000,
4190         0xd28c0001, 0x0001007f,
4191         0xd28d0001, 0x0002027e,
4192         0x10020288, 0xb8810904,
4193         0xb7814000, 0xd1196a01,
4194         0x00000301, 0xbe800087,
4195         0xbefc00c1, 0xd89c4000,
4196         0x00020201, 0xd89cc080,
4197         0x00040401, 0x320202ff,
4198         0x00000800, 0x80808100,
4199         0xbf84fff8, 0x7e020280,
4200         0xbf810000, 0x00000000,
4201 };
4202
4203 static const u32 sgpr_init_compute_shader[] =
4204 {
4205         0xb07c0000, 0xbe8000ff,
4206         0x0000005f, 0xbee50080,
4207         0xbe812c65, 0xbe822c65,
4208         0xbe832c65, 0xbe842c65,
4209         0xbe852c65, 0xb77c0005,
4210         0x80808500, 0xbf84fff8,
4211         0xbe800080, 0xbf810000,
4212 };
4213
4214 static const u32 vgpr_init_compute_shader_arcturus[] = {
4215         0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4216         0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4217         0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4218         0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4219         0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4220         0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4221         0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4222         0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4223         0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4224         0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4225         0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4226         0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4227         0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4228         0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4229         0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4230         0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4231         0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4232         0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4233         0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4234         0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4235         0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4236         0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4237         0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4238         0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4239         0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4240         0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4241         0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4242         0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4243         0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4244         0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4245         0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4246         0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4247         0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4248         0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4249         0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4250         0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4251         0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4252         0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4253         0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4254         0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4255         0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4256         0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4257         0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4258         0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4259         0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4260         0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4261         0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4262         0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4263         0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4264         0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4265         0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4266         0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4267         0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4268         0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4269         0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4270         0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4271         0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4272         0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4273         0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4274         0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4275         0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4276         0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4277         0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4278         0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4279         0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4280         0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4281         0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4282         0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4283         0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4284         0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4285         0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4286         0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4287         0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4288         0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4289         0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4290         0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4291         0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4292         0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4293         0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4294         0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4295         0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4296         0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4297         0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4298         0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4299         0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4300         0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4301         0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4302         0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4303         0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4304         0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4305         0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4306         0xbf84fff8, 0xbf810000,
4307 };
4308
4309 /* When below register arrays changed, please update gpr_reg_size,
4310   and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4311   to cover all gfx9 ASICs */
4312 static const struct soc15_reg_entry vgpr_init_regs[] = {
4313    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4314    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4315    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4316    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4317    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4318    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4319    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4320    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4321    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4322    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4323    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4324    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4325    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4326    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4327 };
4328
4329 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4330    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4331    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4332    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4333    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4334    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4335    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4336    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4337    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4338    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4339    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4340    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4341    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4342    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4343    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4344 };
4345
4346 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4347    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4348    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4349    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4350    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4351    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4352    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4353    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4354    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4355    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4356    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4357    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4358    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4359    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4360    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4361 };
4362
4363 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4364    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4365    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4366    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4367    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4368    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4369    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4370    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4371    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4372    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4373    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4374    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4375    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4376    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4377    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4378 };
4379
4380 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4381    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4382    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4383    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4384    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4385    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4386    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4387    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4388    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4389    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4390    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4391    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4392    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4393    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4394    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4395    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4396    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4397    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4398    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4399    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4400    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4401    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4402    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4403    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4404    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4405    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4406    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4407    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4408    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4409    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4410    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4411    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4412    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4413    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4414 };
4415
4416 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4417 {
4418         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4419         int i, r;
4420
4421         /* only support when RAS is enabled */
4422         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4423                 return 0;
4424
4425         r = amdgpu_ring_alloc(ring, 7);
4426         if (r) {
4427                 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4428                         ring->name, r);
4429                 return r;
4430         }
4431
4432         WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4433         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4434
4435         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4436         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4437                                 PACKET3_DMA_DATA_DST_SEL(1) |
4438                                 PACKET3_DMA_DATA_SRC_SEL(2) |
4439                                 PACKET3_DMA_DATA_ENGINE(0)));
4440         amdgpu_ring_write(ring, 0);
4441         amdgpu_ring_write(ring, 0);
4442         amdgpu_ring_write(ring, 0);
4443         amdgpu_ring_write(ring, 0);
4444         amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4445                                 adev->gds.gds_size);
4446
4447         amdgpu_ring_commit(ring);
4448
4449         for (i = 0; i < adev->usec_timeout; i++) {
4450                 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4451                         break;
4452                 udelay(1);
4453         }
4454
4455         if (i >= adev->usec_timeout)
4456                 r = -ETIMEDOUT;
4457
4458         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4459
4460         return r;
4461 }
4462
4463 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4464 {
4465         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4466         struct amdgpu_ib ib;
4467         struct dma_fence *f = NULL;
4468         int r, i;
4469         unsigned total_size, vgpr_offset, sgpr_offset;
4470         u64 gpu_addr;
4471
4472         int compute_dim_x = adev->gfx.config.max_shader_engines *
4473                                                 adev->gfx.config.max_cu_per_sh *
4474                                                 adev->gfx.config.max_sh_per_se;
4475         int sgpr_work_group_size = 5;
4476         int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4477         int vgpr_init_shader_size;
4478         const u32 *vgpr_init_shader_ptr;
4479         const struct soc15_reg_entry *vgpr_init_regs_ptr;
4480
4481         /* only support when RAS is enabled */
4482         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4483                 return 0;
4484
4485         /* bail if the compute ring is not ready */
4486         if (!ring->sched.ready)
4487                 return 0;
4488
4489         if (adev->asic_type == CHIP_ARCTURUS) {
4490                 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4491                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4492                 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4493         } else {
4494                 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4495                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4496                 vgpr_init_regs_ptr = vgpr_init_regs;
4497         }
4498
4499         total_size =
4500                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4501         total_size +=
4502                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4503         total_size +=
4504                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4505         total_size = ALIGN(total_size, 256);
4506         vgpr_offset = total_size;
4507         total_size += ALIGN(vgpr_init_shader_size, 256);
4508         sgpr_offset = total_size;
4509         total_size += sizeof(sgpr_init_compute_shader);
4510
4511         /* allocate an indirect buffer to put the commands in */
4512         memset(&ib, 0, sizeof(ib));
4513         r = amdgpu_ib_get(adev, NULL, total_size,
4514                                         AMDGPU_IB_POOL_DIRECT, &ib);
4515         if (r) {
4516                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4517                 return r;
4518         }
4519
4520         /* load the compute shaders */
4521         for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4522                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4523
4524         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4525                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4526
4527         /* init the ib length to 0 */
4528         ib.length_dw = 0;
4529
4530         /* VGPR */
4531         /* write the register state for the compute dispatch */
4532         for (i = 0; i < gpr_reg_size; i++) {
4533                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4534                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4535                                                                 - PACKET3_SET_SH_REG_START;
4536                 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4537         }
4538         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4539         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4540         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4541         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4542                                                         - PACKET3_SET_SH_REG_START;
4543         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4544         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4545
4546         /* write dispatch packet */
4547         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4548         ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4549         ib.ptr[ib.length_dw++] = 1; /* y */
4550         ib.ptr[ib.length_dw++] = 1; /* z */
4551         ib.ptr[ib.length_dw++] =
4552                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4553
4554         /* write CS partial flush packet */
4555         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4556         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4557
4558         /* SGPR1 */
4559         /* write the register state for the compute dispatch */
4560         for (i = 0; i < gpr_reg_size; i++) {
4561                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4562                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4563                                                                 - PACKET3_SET_SH_REG_START;
4564                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4565         }
4566         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4567         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4568         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4569         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4570                                                         - PACKET3_SET_SH_REG_START;
4571         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4572         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4573
4574         /* write dispatch packet */
4575         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4576         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4577         ib.ptr[ib.length_dw++] = 1; /* y */
4578         ib.ptr[ib.length_dw++] = 1; /* z */
4579         ib.ptr[ib.length_dw++] =
4580                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4581
4582         /* write CS partial flush packet */
4583         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4584         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4585
4586         /* SGPR2 */
4587         /* write the register state for the compute dispatch */
4588         for (i = 0; i < gpr_reg_size; i++) {
4589                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4590                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4591                                                                 - PACKET3_SET_SH_REG_START;
4592                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4593         }
4594         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4595         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4596         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4597         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4598                                                         - PACKET3_SET_SH_REG_START;
4599         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4600         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4601
4602         /* write dispatch packet */
4603         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4604         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4605         ib.ptr[ib.length_dw++] = 1; /* y */
4606         ib.ptr[ib.length_dw++] = 1; /* z */
4607         ib.ptr[ib.length_dw++] =
4608                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4609
4610         /* write CS partial flush packet */
4611         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4612         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4613
4614         /* shedule the ib on the ring */
4615         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4616         if (r) {
4617                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4618                 goto fail;
4619         }
4620
4621         /* wait for the GPU to finish processing the IB */
4622         r = dma_fence_wait(f, false);
4623         if (r) {
4624                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4625                 goto fail;
4626         }
4627
4628 fail:
4629         amdgpu_ib_free(adev, &ib, NULL);
4630         dma_fence_put(f);
4631
4632         return r;
4633 }
4634
4635 static int gfx_v9_0_early_init(void *handle)
4636 {
4637         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4638
4639         if (adev->asic_type == CHIP_ARCTURUS)
4640                 adev->gfx.num_gfx_rings = 0;
4641         else
4642                 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4643         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4644                                           AMDGPU_MAX_COMPUTE_RINGS);
4645         gfx_v9_0_set_kiq_pm4_funcs(adev);
4646         gfx_v9_0_set_ring_funcs(adev);
4647         gfx_v9_0_set_irq_funcs(adev);
4648         gfx_v9_0_set_gds_init(adev);
4649         gfx_v9_0_set_rlc_funcs(adev);
4650
4651         return 0;
4652 }
4653
4654 static int gfx_v9_0_ecc_late_init(void *handle)
4655 {
4656         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4657         int r;
4658
4659         /*
4660          * Temp workaround to fix the issue that CP firmware fails to
4661          * update read pointer when CPDMA is writing clearing operation
4662          * to GDS in suspend/resume sequence on several cards. So just
4663          * limit this operation in cold boot sequence.
4664          */
4665         if (!adev->in_suspend) {
4666                 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4667                 if (r)
4668                         return r;
4669         }
4670
4671         /* requires IBs so do in late init after IB pool is initialized */
4672         r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4673         if (r)
4674                 return r;
4675
4676         if (adev->gfx.funcs &&
4677             adev->gfx.funcs->reset_ras_error_count)
4678                 adev->gfx.funcs->reset_ras_error_count(adev);
4679
4680         r = amdgpu_gfx_ras_late_init(adev);
4681         if (r)
4682                 return r;
4683
4684         return 0;
4685 }
4686
4687 static int gfx_v9_0_late_init(void *handle)
4688 {
4689         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4690         int r;
4691
4692         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4693         if (r)
4694                 return r;
4695
4696         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4697         if (r)
4698                 return r;
4699
4700         r = gfx_v9_0_ecc_late_init(handle);
4701         if (r)
4702                 return r;
4703
4704         return 0;
4705 }
4706
4707 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4708 {
4709         uint32_t rlc_setting;
4710
4711         /* if RLC is not enabled, do nothing */
4712         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4713         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4714                 return false;
4715
4716         return true;
4717 }
4718
4719 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4720 {
4721         uint32_t data;
4722         unsigned i;
4723
4724         data = RLC_SAFE_MODE__CMD_MASK;
4725         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4726         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4727
4728         /* wait for RLC_SAFE_MODE */
4729         for (i = 0; i < adev->usec_timeout; i++) {
4730                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4731                         break;
4732                 udelay(1);
4733         }
4734 }
4735
4736 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4737 {
4738         uint32_t data;
4739
4740         data = RLC_SAFE_MODE__CMD_MASK;
4741         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4742 }
4743
4744 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4745                                                 bool enable)
4746 {
4747         amdgpu_gfx_rlc_enter_safe_mode(adev);
4748
4749         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4750                 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4751                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4752                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4753         } else {
4754                 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4755                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4756                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4757         }
4758
4759         amdgpu_gfx_rlc_exit_safe_mode(adev);
4760 }
4761
4762 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4763                                                 bool enable)
4764 {
4765         /* TODO: double check if we need to perform under safe mode */
4766         /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4767
4768         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4769                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4770         else
4771                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4772
4773         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4774                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4775         else
4776                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4777
4778         /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4779 }
4780
4781 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4782                                                       bool enable)
4783 {
4784         uint32_t data, def;
4785
4786         amdgpu_gfx_rlc_enter_safe_mode(adev);
4787
4788         /* It is disabled by HW by default */
4789         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4790                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4791                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4792
4793                 if (adev->asic_type != CHIP_VEGA12)
4794                         data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4795
4796                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4797                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4798                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4799
4800                 /* only for Vega10 & Raven1 */
4801                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4802
4803                 if (def != data)
4804                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4805
4806                 /* MGLS is a global flag to control all MGLS in GFX */
4807                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4808                         /* 2 - RLC memory Light sleep */
4809                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4810                                 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4811                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4812                                 if (def != data)
4813                                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4814                         }
4815                         /* 3 - CP memory Light sleep */
4816                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4817                                 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4818                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4819                                 if (def != data)
4820                                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4821                         }
4822                 }
4823         } else {
4824                 /* 1 - MGCG_OVERRIDE */
4825                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4826
4827                 if (adev->asic_type != CHIP_VEGA12)
4828                         data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4829
4830                 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4831                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4832                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4833                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4834
4835                 if (def != data)
4836                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4837
4838                 /* 2 - disable MGLS in RLC */
4839                 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4840                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4841                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4842                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4843                 }
4844
4845                 /* 3 - disable MGLS in CP */
4846                 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4847                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4848                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4849                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4850                 }
4851         }
4852
4853         amdgpu_gfx_rlc_exit_safe_mode(adev);
4854 }
4855
4856 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4857                                            bool enable)
4858 {
4859         uint32_t data, def;
4860
4861         if (adev->asic_type == CHIP_ARCTURUS)
4862                 return;
4863
4864         amdgpu_gfx_rlc_enter_safe_mode(adev);
4865
4866         /* Enable 3D CGCG/CGLS */
4867         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
4868                 /* write cmd to clear cgcg/cgls ov */
4869                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4870                 /* unset CGCG override */
4871                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4872                 /* update CGCG and CGLS override bits */
4873                 if (def != data)
4874                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4875
4876                 /* enable 3Dcgcg FSM(0x0000363f) */
4877                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4878
4879                 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4880                         RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4881                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4882                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4883                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4884                 if (def != data)
4885                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4886
4887                 /* set IDLE_POLL_COUNT(0x00900100) */
4888                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4889                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4890                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4891                 if (def != data)
4892                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4893         } else {
4894                 /* Disable CGCG/CGLS */
4895                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4896                 /* disable cgcg, cgls should be disabled */
4897                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4898                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4899                 /* disable cgcg and cgls in FSM */
4900                 if (def != data)
4901                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4902         }
4903
4904         amdgpu_gfx_rlc_exit_safe_mode(adev);
4905 }
4906
4907 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4908                                                       bool enable)
4909 {
4910         uint32_t def, data;
4911
4912         amdgpu_gfx_rlc_enter_safe_mode(adev);
4913
4914         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4915                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4916                 /* unset CGCG override */
4917                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4918                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4919                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4920                 else
4921                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4922                 /* update CGCG and CGLS override bits */
4923                 if (def != data)
4924                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4925
4926                 /* enable cgcg FSM(0x0000363F) */
4927                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4928
4929                 if (adev->asic_type == CHIP_ARCTURUS)
4930                         data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4931                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4932                 else
4933                         data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4934                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4935                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4936                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4937                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4938                 if (def != data)
4939                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4940
4941                 /* set IDLE_POLL_COUNT(0x00900100) */
4942                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4943                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4944                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4945                 if (def != data)
4946                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4947         } else {
4948                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4949                 /* reset CGCG/CGLS bits */
4950                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
4951                 /* disable cgcg and cgls in FSM */
4952                 if (def != data)
4953                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4954         }
4955
4956         amdgpu_gfx_rlc_exit_safe_mode(adev);
4957 }
4958
4959 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4960                                             bool enable)
4961 {
4962         if (enable) {
4963                 /* CGCG/CGLS should be enabled after MGCG/MGLS
4964                  * ===  MGCG + MGLS ===
4965                  */
4966                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4967                 /* ===  CGCG /CGLS for GFX 3D Only === */
4968                 gfx_v9_0_update_3d_clock_gating(adev, enable);
4969                 /* ===  CGCG + CGLS === */
4970                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4971         } else {
4972                 /* CGCG/CGLS should be disabled before MGCG/MGLS
4973                  * ===  CGCG + CGLS ===
4974                  */
4975                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4976                 /* ===  CGCG /CGLS for GFX 3D Only === */
4977                 gfx_v9_0_update_3d_clock_gating(adev, enable);
4978                 /* ===  MGCG + MGLS === */
4979                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4980         }
4981         return 0;
4982 }
4983
4984 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
4985 {
4986         u32 reg, data;
4987
4988         reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
4989         if (amdgpu_sriov_is_pp_one_vf(adev))
4990                 data = RREG32_NO_KIQ(reg);
4991         else
4992                 data = RREG32(reg);
4993
4994         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
4995         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
4996
4997         if (amdgpu_sriov_is_pp_one_vf(adev))
4998                 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
4999         else
5000                 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5001 }
5002
5003 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5004                                         uint32_t offset,
5005                                         struct soc15_reg_rlcg *entries, int arr_size)
5006 {
5007         int i;
5008         uint32_t reg;
5009
5010         if (!entries)
5011                 return false;
5012
5013         for (i = 0; i < arr_size; i++) {
5014                 const struct soc15_reg_rlcg *entry;
5015
5016                 entry = &entries[i];
5017                 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5018                 if (offset == reg)
5019                         return true;
5020         }
5021
5022         return false;
5023 }
5024
5025 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5026 {
5027         return gfx_v9_0_check_rlcg_range(adev, offset,
5028                                         (void *)rlcg_access_gc_9_0,
5029                                         ARRAY_SIZE(rlcg_access_gc_9_0));
5030 }
5031
5032 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5033         .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5034         .set_safe_mode = gfx_v9_0_set_safe_mode,
5035         .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5036         .init = gfx_v9_0_rlc_init,
5037         .get_csb_size = gfx_v9_0_get_csb_size,
5038         .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5039         .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5040         .resume = gfx_v9_0_rlc_resume,
5041         .stop = gfx_v9_0_rlc_stop,
5042         .reset = gfx_v9_0_rlc_reset,
5043         .start = gfx_v9_0_rlc_start,
5044         .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5045         .rlcg_wreg = gfx_v9_0_rlcg_wreg,
5046         .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5047 };
5048
5049 static int gfx_v9_0_set_powergating_state(void *handle,
5050                                           enum amd_powergating_state state)
5051 {
5052         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5053         bool enable = (state == AMD_PG_STATE_GATE);
5054
5055         switch (adev->asic_type) {
5056         case CHIP_RAVEN:
5057         case CHIP_RENOIR:
5058                 if (!enable)
5059                         amdgpu_gfx_off_ctrl(adev, false);
5060
5061                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5062                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5063                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5064                 } else {
5065                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5066                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5067                 }
5068
5069                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5070                         gfx_v9_0_enable_cp_power_gating(adev, true);
5071                 else
5072                         gfx_v9_0_enable_cp_power_gating(adev, false);
5073
5074                 /* update gfx cgpg state */
5075                 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5076
5077                 /* update mgcg state */
5078                 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5079
5080                 if (enable)
5081                         amdgpu_gfx_off_ctrl(adev, true);
5082                 break;
5083         case CHIP_VEGA12:
5084                 amdgpu_gfx_off_ctrl(adev, enable);
5085                 break;
5086         default:
5087                 break;
5088         }
5089
5090         return 0;
5091 }
5092
5093 static int gfx_v9_0_set_clockgating_state(void *handle,
5094                                           enum amd_clockgating_state state)
5095 {
5096         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5097
5098         if (amdgpu_sriov_vf(adev))
5099                 return 0;
5100
5101         switch (adev->asic_type) {
5102         case CHIP_VEGA10:
5103         case CHIP_VEGA12:
5104         case CHIP_VEGA20:
5105         case CHIP_RAVEN:
5106         case CHIP_ARCTURUS:
5107         case CHIP_RENOIR:
5108                 gfx_v9_0_update_gfx_clock_gating(adev,
5109                                                  state == AMD_CG_STATE_GATE);
5110                 break;
5111         default:
5112                 break;
5113         }
5114         return 0;
5115 }
5116
5117 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
5118 {
5119         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5120         int data;
5121
5122         if (amdgpu_sriov_vf(adev))
5123                 *flags = 0;
5124
5125         /* AMD_CG_SUPPORT_GFX_MGCG */
5126         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5127         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5128                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5129
5130         /* AMD_CG_SUPPORT_GFX_CGCG */
5131         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5132         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5133                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5134
5135         /* AMD_CG_SUPPORT_GFX_CGLS */
5136         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5137                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5138
5139         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5140         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5141         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5142                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5143
5144         /* AMD_CG_SUPPORT_GFX_CP_LS */
5145         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5146         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5147                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5148
5149         if (adev->asic_type != CHIP_ARCTURUS) {
5150                 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5151                 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5152                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5153                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5154
5155                 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5156                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5157                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5158         }
5159 }
5160
5161 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5162 {
5163         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
5164 }
5165
5166 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5167 {
5168         struct amdgpu_device *adev = ring->adev;
5169         u64 wptr;
5170
5171         /* XXX check if swapping is necessary on BE */
5172         if (ring->use_doorbell) {
5173                 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
5174         } else {
5175                 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5176                 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5177         }
5178
5179         return wptr;
5180 }
5181
5182 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5183 {
5184         struct amdgpu_device *adev = ring->adev;
5185
5186         if (ring->use_doorbell) {
5187                 /* XXX check if swapping is necessary on BE */
5188                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5189                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5190         } else {
5191                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5192                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5193         }
5194 }
5195
5196 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5197 {
5198         struct amdgpu_device *adev = ring->adev;
5199         u32 ref_and_mask, reg_mem_engine;
5200         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5201
5202         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5203                 switch (ring->me) {
5204                 case 1:
5205                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5206                         break;
5207                 case 2:
5208                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5209                         break;
5210                 default:
5211                         return;
5212                 }
5213                 reg_mem_engine = 0;
5214         } else {
5215                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5216                 reg_mem_engine = 1; /* pfp */
5217         }
5218
5219         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5220                               adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5221                               adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5222                               ref_and_mask, ref_and_mask, 0x20);
5223 }
5224
5225 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5226                                         struct amdgpu_job *job,
5227                                         struct amdgpu_ib *ib,
5228                                         uint32_t flags)
5229 {
5230         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5231         u32 header, control = 0;
5232
5233         if (ib->flags & AMDGPU_IB_FLAG_CE)
5234                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5235         else
5236                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5237
5238         control |= ib->length_dw | (vmid << 24);
5239
5240         if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5241                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5242
5243                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5244                         gfx_v9_0_ring_emit_de_meta(ring);
5245         }
5246
5247         amdgpu_ring_write(ring, header);
5248         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5249         amdgpu_ring_write(ring,
5250 #ifdef __BIG_ENDIAN
5251                 (2 << 0) |
5252 #endif
5253                 lower_32_bits(ib->gpu_addr));
5254         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5255         amdgpu_ring_write(ring, control);
5256 }
5257
5258 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5259                                           struct amdgpu_job *job,
5260                                           struct amdgpu_ib *ib,
5261                                           uint32_t flags)
5262 {
5263         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5264         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5265
5266         /* Currently, there is a high possibility to get wave ID mismatch
5267          * between ME and GDS, leading to a hw deadlock, because ME generates
5268          * different wave IDs than the GDS expects. This situation happens
5269          * randomly when at least 5 compute pipes use GDS ordered append.
5270          * The wave IDs generated by ME are also wrong after suspend/resume.
5271          * Those are probably bugs somewhere else in the kernel driver.
5272          *
5273          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5274          * GDS to 0 for this ring (me/pipe).
5275          */
5276         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5277                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5278                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5279                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5280         }
5281
5282         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5283         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5284         amdgpu_ring_write(ring,
5285 #ifdef __BIG_ENDIAN
5286                                 (2 << 0) |
5287 #endif
5288                                 lower_32_bits(ib->gpu_addr));
5289         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5290         amdgpu_ring_write(ring, control);
5291 }
5292
5293 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5294                                      u64 seq, unsigned flags)
5295 {
5296         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5297         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5298         bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5299
5300         /* RELEASE_MEM - flush caches, send int */
5301         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5302         amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
5303                                                EOP_TC_NC_ACTION_EN) :
5304                                               (EOP_TCL1_ACTION_EN |
5305                                                EOP_TC_ACTION_EN |
5306                                                EOP_TC_WB_ACTION_EN |
5307                                                EOP_TC_MD_ACTION_EN)) |
5308                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5309                                  EVENT_INDEX(5)));
5310         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5311
5312         /*
5313          * the address should be Qword aligned if 64bit write, Dword
5314          * aligned if only send 32bit data low (discard data high)
5315          */
5316         if (write64bit)
5317                 BUG_ON(addr & 0x7);
5318         else
5319                 BUG_ON(addr & 0x3);
5320         amdgpu_ring_write(ring, lower_32_bits(addr));
5321         amdgpu_ring_write(ring, upper_32_bits(addr));
5322         amdgpu_ring_write(ring, lower_32_bits(seq));
5323         amdgpu_ring_write(ring, upper_32_bits(seq));
5324         amdgpu_ring_write(ring, 0);
5325 }
5326
5327 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5328 {
5329         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5330         uint32_t seq = ring->fence_drv.sync_seq;
5331         uint64_t addr = ring->fence_drv.gpu_addr;
5332
5333         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5334                               lower_32_bits(addr), upper_32_bits(addr),
5335                               seq, 0xffffffff, 4);
5336 }
5337
5338 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5339                                         unsigned vmid, uint64_t pd_addr)
5340 {
5341         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5342
5343         /* compute doesn't have PFP */
5344         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5345                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5346                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5347                 amdgpu_ring_write(ring, 0x0);
5348         }
5349 }
5350
5351 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5352 {
5353         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
5354 }
5355
5356 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5357 {
5358         u64 wptr;
5359
5360         /* XXX check if swapping is necessary on BE */
5361         if (ring->use_doorbell)
5362                 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
5363         else
5364                 BUG();
5365         return wptr;
5366 }
5367
5368 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5369 {
5370         struct amdgpu_device *adev = ring->adev;
5371
5372         /* XXX check if swapping is necessary on BE */
5373         if (ring->use_doorbell) {
5374                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5375                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5376         } else{
5377                 BUG(); /* only DOORBELL method supported on gfx9 now */
5378         }
5379 }
5380
5381 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5382                                          u64 seq, unsigned int flags)
5383 {
5384         struct amdgpu_device *adev = ring->adev;
5385
5386         /* we only allocate 32bit for each seq wb address */
5387         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5388
5389         /* write fence seq to the "addr" */
5390         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5391         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5392                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5393         amdgpu_ring_write(ring, lower_32_bits(addr));
5394         amdgpu_ring_write(ring, upper_32_bits(addr));
5395         amdgpu_ring_write(ring, lower_32_bits(seq));
5396
5397         if (flags & AMDGPU_FENCE_FLAG_INT) {
5398                 /* set register to trigger INT */
5399                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5400                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5401                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5402                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5403                 amdgpu_ring_write(ring, 0);
5404                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5405         }
5406 }
5407
5408 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5409 {
5410         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5411         amdgpu_ring_write(ring, 0);
5412 }
5413
5414 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
5415 {
5416         struct v9_ce_ib_state ce_payload = {0};
5417         uint64_t csa_addr;
5418         int cnt;
5419
5420         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5421         csa_addr = amdgpu_csa_vaddr(ring->adev);
5422
5423         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5424         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5425                                  WRITE_DATA_DST_SEL(8) |
5426                                  WR_CONFIRM) |
5427                                  WRITE_DATA_CACHE_POLICY(0));
5428         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5429         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5430         amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
5431 }
5432
5433 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
5434 {
5435         struct v9_de_ib_state de_payload = {0};
5436         uint64_t csa_addr, gds_addr;
5437         int cnt;
5438
5439         csa_addr = amdgpu_csa_vaddr(ring->adev);
5440         gds_addr = csa_addr + 4096;
5441         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5442         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5443
5444         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5445         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5446         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5447                                  WRITE_DATA_DST_SEL(8) |
5448                                  WR_CONFIRM) |
5449                                  WRITE_DATA_CACHE_POLICY(0));
5450         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5451         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5452         amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
5453 }
5454
5455 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5456                                    bool secure)
5457 {
5458         uint32_t v = secure ? FRAME_TMZ : 0;
5459
5460         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5461         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5462 }
5463
5464 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5465 {
5466         uint32_t dw2 = 0;
5467
5468         if (amdgpu_sriov_vf(ring->adev))
5469                 gfx_v9_0_ring_emit_ce_meta(ring);
5470
5471         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5472         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5473                 /* set load_global_config & load_global_uconfig */
5474                 dw2 |= 0x8001;
5475                 /* set load_cs_sh_regs */
5476                 dw2 |= 0x01000000;
5477                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5478                 dw2 |= 0x10002;
5479
5480                 /* set load_ce_ram if preamble presented */
5481                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5482                         dw2 |= 0x10000000;
5483         } else {
5484                 /* still load_ce_ram if this is the first time preamble presented
5485                  * although there is no context switch happens.
5486                  */
5487                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5488                         dw2 |= 0x10000000;
5489         }
5490
5491         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5492         amdgpu_ring_write(ring, dw2);
5493         amdgpu_ring_write(ring, 0);
5494 }
5495
5496 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5497 {
5498         unsigned ret;
5499         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5500         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5501         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5502         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5503         ret = ring->wptr & ring->buf_mask;
5504         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5505         return ret;
5506 }
5507
5508 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5509 {
5510         unsigned cur;
5511         BUG_ON(offset > ring->buf_mask);
5512         BUG_ON(ring->ring[offset] != 0x55aa55aa);
5513
5514         cur = (ring->wptr & ring->buf_mask) - 1;
5515         if (likely(cur > offset))
5516                 ring->ring[offset] = cur - offset;
5517         else
5518                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5519 }
5520
5521 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5522                                     uint32_t reg_val_offs)
5523 {
5524         struct amdgpu_device *adev = ring->adev;
5525
5526         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5527         amdgpu_ring_write(ring, 0 |     /* src: register*/
5528                                 (5 << 8) |      /* dst: memory */
5529                                 (1 << 20));     /* write confirm */
5530         amdgpu_ring_write(ring, reg);
5531         amdgpu_ring_write(ring, 0);
5532         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5533                                 reg_val_offs * 4));
5534         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5535                                 reg_val_offs * 4));
5536 }
5537
5538 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5539                                     uint32_t val)
5540 {
5541         uint32_t cmd = 0;
5542
5543         switch (ring->funcs->type) {
5544         case AMDGPU_RING_TYPE_GFX:
5545                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5546                 break;
5547         case AMDGPU_RING_TYPE_KIQ:
5548                 cmd = (1 << 16); /* no inc addr */
5549                 break;
5550         default:
5551                 cmd = WR_CONFIRM;
5552                 break;
5553         }
5554         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5555         amdgpu_ring_write(ring, cmd);
5556         amdgpu_ring_write(ring, reg);
5557         amdgpu_ring_write(ring, 0);
5558         amdgpu_ring_write(ring, val);
5559 }
5560
5561 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5562                                         uint32_t val, uint32_t mask)
5563 {
5564         gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5565 }
5566
5567 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5568                                                   uint32_t reg0, uint32_t reg1,
5569                                                   uint32_t ref, uint32_t mask)
5570 {
5571         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5572         struct amdgpu_device *adev = ring->adev;
5573         bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5574                 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5575
5576         if (fw_version_ok)
5577                 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5578                                       ref, mask, 0x20);
5579         else
5580                 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5581                                                            ref, mask);
5582 }
5583
5584 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5585 {
5586         struct amdgpu_device *adev = ring->adev;
5587         uint32_t value = 0;
5588
5589         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5590         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5591         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5592         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5593         WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5594 }
5595
5596 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5597                                                  enum amdgpu_interrupt_state state)
5598 {
5599         switch (state) {
5600         case AMDGPU_IRQ_STATE_DISABLE:
5601         case AMDGPU_IRQ_STATE_ENABLE:
5602                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5603                                TIME_STAMP_INT_ENABLE,
5604                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5605                 break;
5606         default:
5607                 break;
5608         }
5609 }
5610
5611 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5612                                                      int me, int pipe,
5613                                                      enum amdgpu_interrupt_state state)
5614 {
5615         u32 mec_int_cntl, mec_int_cntl_reg;
5616
5617         /*
5618          * amdgpu controls only the first MEC. That's why this function only
5619          * handles the setting of interrupts for this specific MEC. All other
5620          * pipes' interrupts are set by amdkfd.
5621          */
5622
5623         if (me == 1) {
5624                 switch (pipe) {
5625                 case 0:
5626                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5627                         break;
5628                 case 1:
5629                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5630                         break;
5631                 case 2:
5632                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5633                         break;
5634                 case 3:
5635                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5636                         break;
5637                 default:
5638                         DRM_DEBUG("invalid pipe %d\n", pipe);
5639                         return;
5640                 }
5641         } else {
5642                 DRM_DEBUG("invalid me %d\n", me);
5643                 return;
5644         }
5645
5646         switch (state) {
5647         case AMDGPU_IRQ_STATE_DISABLE:
5648                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5649                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5650                                              TIME_STAMP_INT_ENABLE, 0);
5651                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5652                 break;
5653         case AMDGPU_IRQ_STATE_ENABLE:
5654                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5655                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5656                                              TIME_STAMP_INT_ENABLE, 1);
5657                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5658                 break;
5659         default:
5660                 break;
5661         }
5662 }
5663
5664 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5665                                              struct amdgpu_irq_src *source,
5666                                              unsigned type,
5667                                              enum amdgpu_interrupt_state state)
5668 {
5669         switch (state) {
5670         case AMDGPU_IRQ_STATE_DISABLE:
5671         case AMDGPU_IRQ_STATE_ENABLE:
5672                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5673                                PRIV_REG_INT_ENABLE,
5674                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5675                 break;
5676         default:
5677                 break;
5678         }
5679
5680         return 0;
5681 }
5682
5683 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5684                                               struct amdgpu_irq_src *source,
5685                                               unsigned type,
5686                                               enum amdgpu_interrupt_state state)
5687 {
5688         switch (state) {
5689         case AMDGPU_IRQ_STATE_DISABLE:
5690         case AMDGPU_IRQ_STATE_ENABLE:
5691                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5692                                PRIV_INSTR_INT_ENABLE,
5693                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5694                 break;
5695         default:
5696                 break;
5697         }
5698
5699         return 0;
5700 }
5701
5702 #define ENABLE_ECC_ON_ME_PIPE(me, pipe)                         \
5703         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5704                         CP_ECC_ERROR_INT_ENABLE, 1)
5705
5706 #define DISABLE_ECC_ON_ME_PIPE(me, pipe)                        \
5707         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5708                         CP_ECC_ERROR_INT_ENABLE, 0)
5709
5710 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5711                                               struct amdgpu_irq_src *source,
5712                                               unsigned type,
5713                                               enum amdgpu_interrupt_state state)
5714 {
5715         switch (state) {
5716         case AMDGPU_IRQ_STATE_DISABLE:
5717                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5718                                 CP_ECC_ERROR_INT_ENABLE, 0);
5719                 DISABLE_ECC_ON_ME_PIPE(1, 0);
5720                 DISABLE_ECC_ON_ME_PIPE(1, 1);
5721                 DISABLE_ECC_ON_ME_PIPE(1, 2);
5722                 DISABLE_ECC_ON_ME_PIPE(1, 3);
5723                 break;
5724
5725         case AMDGPU_IRQ_STATE_ENABLE:
5726                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5727                                 CP_ECC_ERROR_INT_ENABLE, 1);
5728                 ENABLE_ECC_ON_ME_PIPE(1, 0);
5729                 ENABLE_ECC_ON_ME_PIPE(1, 1);
5730                 ENABLE_ECC_ON_ME_PIPE(1, 2);
5731                 ENABLE_ECC_ON_ME_PIPE(1, 3);
5732                 break;
5733         default:
5734                 break;
5735         }
5736
5737         return 0;
5738 }
5739
5740
5741 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5742                                             struct amdgpu_irq_src *src,
5743                                             unsigned type,
5744                                             enum amdgpu_interrupt_state state)
5745 {
5746         switch (type) {
5747         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5748                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5749                 break;
5750         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5751                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5752                 break;
5753         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5754                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5755                 break;
5756         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5757                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5758                 break;
5759         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5760                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5761                 break;
5762         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5763                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5764                 break;
5765         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5766                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5767                 break;
5768         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5769                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5770                 break;
5771         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5772                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5773                 break;
5774         default:
5775                 break;
5776         }
5777         return 0;
5778 }
5779
5780 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5781                             struct amdgpu_irq_src *source,
5782                             struct amdgpu_iv_entry *entry)
5783 {
5784         int i;
5785         u8 me_id, pipe_id, queue_id;
5786         struct amdgpu_ring *ring;
5787
5788         DRM_DEBUG("IH: CP EOP\n");
5789         me_id = (entry->ring_id & 0x0c) >> 2;
5790         pipe_id = (entry->ring_id & 0x03) >> 0;
5791         queue_id = (entry->ring_id & 0x70) >> 4;
5792
5793         switch (me_id) {
5794         case 0:
5795                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5796                 break;
5797         case 1:
5798         case 2:
5799                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5800                         ring = &adev->gfx.compute_ring[i];
5801                         /* Per-queue interrupt is supported for MEC starting from VI.
5802                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
5803                           */
5804                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5805                                 amdgpu_fence_process(ring);
5806                 }
5807                 break;
5808         }
5809         return 0;
5810 }
5811
5812 static void gfx_v9_0_fault(struct amdgpu_device *adev,
5813                            struct amdgpu_iv_entry *entry)
5814 {
5815         u8 me_id, pipe_id, queue_id;
5816         struct amdgpu_ring *ring;
5817         int i;
5818
5819         me_id = (entry->ring_id & 0x0c) >> 2;
5820         pipe_id = (entry->ring_id & 0x03) >> 0;
5821         queue_id = (entry->ring_id & 0x70) >> 4;
5822
5823         switch (me_id) {
5824         case 0:
5825                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5826                 break;
5827         case 1:
5828         case 2:
5829                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5830                         ring = &adev->gfx.compute_ring[i];
5831                         if (ring->me == me_id && ring->pipe == pipe_id &&
5832                             ring->queue == queue_id)
5833                                 drm_sched_fault(&ring->sched);
5834                 }
5835                 break;
5836         }
5837 }
5838
5839 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5840                                  struct amdgpu_irq_src *source,
5841                                  struct amdgpu_iv_entry *entry)
5842 {
5843         DRM_ERROR("Illegal register access in command stream\n");
5844         gfx_v9_0_fault(adev, entry);
5845         return 0;
5846 }
5847
5848 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5849                                   struct amdgpu_irq_src *source,
5850                                   struct amdgpu_iv_entry *entry)
5851 {
5852         DRM_ERROR("Illegal instruction in command stream\n");
5853         gfx_v9_0_fault(adev, entry);
5854         return 0;
5855 }
5856
5857
5858 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5859         { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5860           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5861           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5862         },
5863         { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5864           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5865           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5866         },
5867         { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5868           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5869           0, 0
5870         },
5871         { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5872           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5873           0, 0
5874         },
5875         { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5876           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5877           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5878         },
5879         { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5880           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5881           0, 0
5882         },
5883         { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5884           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
5885           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
5886         },
5887         { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
5888           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
5889           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
5890         },
5891         { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
5892           SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
5893           0, 0
5894         },
5895         { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
5896           SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
5897           0, 0
5898         },
5899         { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
5900           SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
5901           0, 0
5902         },
5903         { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5904           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
5905           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
5906         },
5907         { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5908           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
5909           0, 0
5910         },
5911         { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5912           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
5913           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
5914         },
5915         { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
5916           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5917           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
5918           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
5919         },
5920         { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
5921           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5922           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
5923           0, 0
5924         },
5925         { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
5926           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5927           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
5928           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
5929         },
5930         { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
5931           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5932           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
5933           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
5934         },
5935         { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
5936           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5937           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
5938           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
5939         },
5940         { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
5941           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5942           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
5943           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
5944         },
5945         { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
5946           SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
5947           0, 0
5948         },
5949         { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5950           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
5951           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
5952         },
5953         { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5954           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
5955           0, 0
5956         },
5957         { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5958           SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
5959           0, 0
5960         },
5961         { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5962           SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
5963           0, 0
5964         },
5965         { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5966           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
5967           0, 0
5968         },
5969         { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5970           SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
5971           0, 0
5972         },
5973         { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5974           SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
5975           0, 0
5976         },
5977         { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5978           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
5979           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
5980         },
5981         { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5982           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
5983           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
5984         },
5985         { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5986           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
5987           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
5988         },
5989         { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5990           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
5991           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
5992         },
5993         { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5994           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
5995           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
5996         },
5997         { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5998           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
5999           0, 0
6000         },
6001         { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6002           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6003           0, 0
6004         },
6005         { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6006           SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6007           0, 0
6008         },
6009         { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6010           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6011           0, 0
6012         },
6013         { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6014           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6015           0, 0
6016         },
6017         { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6018           SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6019           0, 0
6020         },
6021         { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6022           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6023           0, 0
6024         },
6025         { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6026           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6027           0, 0
6028         },
6029         { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6030           SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6031           0, 0
6032         },
6033         { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6034           SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6035           0, 0
6036         },
6037         { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6038           SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6039           0, 0
6040         },
6041         { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6042           SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6043           0, 0
6044         },
6045         { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6046           SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6047           0, 0
6048         },
6049         { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6050           SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6051           0, 0
6052         },
6053         { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6054           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6055           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6056         },
6057         { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6058           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6059           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6060         },
6061         { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6062           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6063           0, 0
6064         },
6065         { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6066           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6067           0, 0
6068         },
6069         { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6070           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6071           0, 0
6072         },
6073         { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6074           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6075           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6076         },
6077         { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6078           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6079           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6080         },
6081         { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6082           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6083           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6084         },
6085         { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6086           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6087           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6088         },
6089         { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6090           SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6091           0, 0
6092         },
6093         { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6094           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6095           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6096         },
6097         { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6098           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6099           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6100         },
6101         { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6102           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6103           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6104         },
6105         { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6106           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6107           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6108         },
6109         { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6110           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6111           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6112         },
6113         { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6114           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6115           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6116         },
6117         { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6118           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6119           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6120         },
6121         { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6122           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6123           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6124         },
6125         { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6126           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6127           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6128         },
6129         { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6130           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6131           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6132         },
6133         { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6134           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6135           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6136         },
6137         { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6138           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6139           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6140         },
6141         { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6142           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6143           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6144         },
6145         { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6146           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6147           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6148         },
6149         { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6150           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6151           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6152         },
6153         { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6154           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6155           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6156         },
6157         { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6158           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6159           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6160         },
6161         { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6162           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6163           0, 0
6164         },
6165         { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6166           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6167           0, 0
6168         },
6169         { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6170           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6171           0, 0
6172         },
6173         { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6174           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6175           0, 0
6176         },
6177         { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6178           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6179           0, 0
6180         },
6181         { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6182           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6183           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6184         },
6185         { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6186           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6187           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6188         },
6189         { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6190           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6191           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6192         },
6193         { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6194           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6195           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6196         },
6197         { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6198           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6199           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6200         },
6201         { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6202           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6203           0, 0
6204         },
6205         { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6206           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6207           0, 0
6208         },
6209         { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6210           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6211           0, 0
6212         },
6213         { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6214           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6215           0, 0
6216         },
6217         { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6218           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6219           0, 0
6220         },
6221         { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6222           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6223           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6224         },
6225         { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6226           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6227           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6228         },
6229         { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6230           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6231           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6232         },
6233         { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6234           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6235           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6236         },
6237         { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6238           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6239           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6240         },
6241         { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6242           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6243           0, 0
6244         },
6245         { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6246           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6247           0, 0
6248         },
6249         { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6250           SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6251           0, 0
6252         },
6253         { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6254           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6255           0, 0
6256         },
6257         { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6258           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6259           0, 0
6260         },
6261         { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6262           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6263           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6264         },
6265         { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6266           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6267           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6268         },
6269         { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6270           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6271           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6272         },
6273         { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6274           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6275           0, 0
6276         },
6277         { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6278           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6279           0, 0
6280         },
6281         { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6282           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6283           0, 0
6284         },
6285         { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6286           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6287           0, 0
6288         },
6289         { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6290           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6291           0, 0
6292         },
6293         { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6294           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6295           0, 0
6296         }
6297 };
6298
6299 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6300                                      void *inject_if)
6301 {
6302         struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6303         int ret;
6304         struct ta_ras_trigger_error_input block_info = { 0 };
6305
6306         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6307                 return -EINVAL;
6308
6309         if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6310                 return -EINVAL;
6311
6312         if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6313                 return -EPERM;
6314
6315         if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6316               info->head.type)) {
6317                 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6318                         ras_gfx_subblocks[info->head.sub_block_index].name,
6319                         info->head.type);
6320                 return -EPERM;
6321         }
6322
6323         if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6324               info->head.type)) {
6325                 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6326                         ras_gfx_subblocks[info->head.sub_block_index].name,
6327                         info->head.type);
6328                 return -EPERM;
6329         }
6330
6331         block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6332         block_info.sub_block_index =
6333                 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6334         block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6335         block_info.address = info->address;
6336         block_info.value = info->value;
6337
6338         mutex_lock(&adev->grbm_idx_mutex);
6339         ret = psp_ras_trigger_error(&adev->psp, &block_info);
6340         mutex_unlock(&adev->grbm_idx_mutex);
6341
6342         return ret;
6343 }
6344
6345 static const char *vml2_mems[] = {
6346         "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6347         "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6348         "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6349         "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6350         "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6351         "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6352         "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6353         "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6354         "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6355         "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6356         "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6357         "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6358         "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6359         "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6360         "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6361         "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6362 };
6363
6364 static const char *vml2_walker_mems[] = {
6365         "UTC_VML2_CACHE_PDE0_MEM0",
6366         "UTC_VML2_CACHE_PDE0_MEM1",
6367         "UTC_VML2_CACHE_PDE1_MEM0",
6368         "UTC_VML2_CACHE_PDE1_MEM1",
6369         "UTC_VML2_CACHE_PDE2_MEM0",
6370         "UTC_VML2_CACHE_PDE2_MEM1",
6371         "UTC_VML2_RDIF_LOG_FIFO",
6372 };
6373
6374 static const char *atc_l2_cache_2m_mems[] = {
6375         "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6376         "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6377         "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6378         "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6379 };
6380
6381 static const char *atc_l2_cache_4k_mems[] = {
6382         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6383         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6384         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6385         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6386         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6387         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6388         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6389         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6390         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6391         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6392         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6393         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6394         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6395         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6396         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6397         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6398         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6399         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6400         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6401         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6402         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6403         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6404         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6405         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6406         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6407         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6408         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6409         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6410         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6411         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6412         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6413         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6414 };
6415
6416 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6417                                          struct ras_err_data *err_data)
6418 {
6419         uint32_t i, data;
6420         uint32_t sec_count, ded_count;
6421
6422         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6423         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6424         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6425         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6426         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6427         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6428         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6429         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6430
6431         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6432                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6433                 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6434
6435                 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6436                 if (sec_count) {
6437                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6438                                 "SEC %d\n", i, vml2_mems[i], sec_count);
6439                         err_data->ce_count += sec_count;
6440                 }
6441
6442                 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6443                 if (ded_count) {
6444                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6445                                 "DED %d\n", i, vml2_mems[i], ded_count);
6446                         err_data->ue_count += ded_count;
6447                 }
6448         }
6449
6450         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6451                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6452                 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6453
6454                 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6455                                                 SEC_COUNT);
6456                 if (sec_count) {
6457                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6458                                 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6459                         err_data->ce_count += sec_count;
6460                 }
6461
6462                 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6463                                                 DED_COUNT);
6464                 if (ded_count) {
6465                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6466                                 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6467                         err_data->ue_count += ded_count;
6468                 }
6469         }
6470
6471         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6472                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6473                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6474
6475                 sec_count = (data & 0x00006000L) >> 0xd;
6476                 if (sec_count) {
6477                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6478                                 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6479                                 sec_count);
6480                         err_data->ce_count += sec_count;
6481                 }
6482         }
6483
6484         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6485                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6486                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6487
6488                 sec_count = (data & 0x00006000L) >> 0xd;
6489                 if (sec_count) {
6490                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6491                                 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6492                                 sec_count);
6493                         err_data->ce_count += sec_count;
6494                 }
6495
6496                 ded_count = (data & 0x00018000L) >> 0xf;
6497                 if (ded_count) {
6498                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6499                                 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6500                                 ded_count);
6501                         err_data->ue_count += ded_count;
6502                 }
6503         }
6504
6505         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6506         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6507         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6508         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6509
6510         return 0;
6511 }
6512
6513 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6514         const struct soc15_reg_entry *reg,
6515         uint32_t se_id, uint32_t inst_id, uint32_t value,
6516         uint32_t *sec_count, uint32_t *ded_count)
6517 {
6518         uint32_t i;
6519         uint32_t sec_cnt, ded_cnt;
6520
6521         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6522                 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6523                         gfx_v9_0_ras_fields[i].seg != reg->seg ||
6524                         gfx_v9_0_ras_fields[i].inst != reg->inst)
6525                         continue;
6526
6527                 sec_cnt = (value &
6528                                 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6529                                 gfx_v9_0_ras_fields[i].sec_count_shift;
6530                 if (sec_cnt) {
6531                         dev_info(adev->dev, "GFX SubBlock %s, "
6532                                 "Instance[%d][%d], SEC %d\n",
6533                                 gfx_v9_0_ras_fields[i].name,
6534                                 se_id, inst_id,
6535                                 sec_cnt);
6536                         *sec_count += sec_cnt;
6537                 }
6538
6539                 ded_cnt = (value &
6540                                 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6541                                 gfx_v9_0_ras_fields[i].ded_count_shift;
6542                 if (ded_cnt) {
6543                         dev_info(adev->dev, "GFX SubBlock %s, "
6544                                 "Instance[%d][%d], DED %d\n",
6545                                 gfx_v9_0_ras_fields[i].name,
6546                                 se_id, inst_id,
6547                                 ded_cnt);
6548                         *ded_count += ded_cnt;
6549                 }
6550         }
6551
6552         return 0;
6553 }
6554
6555 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6556 {
6557         int i, j, k;
6558
6559         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6560                 return;
6561
6562         /* read back registers to clear the counters */
6563         mutex_lock(&adev->grbm_idx_mutex);
6564         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6565                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6566                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6567                                 gfx_v9_0_select_se_sh(adev, j, 0x0, k);
6568                                 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6569                         }
6570                 }
6571         }
6572         WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6573         mutex_unlock(&adev->grbm_idx_mutex);
6574
6575         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6576         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6577         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6578         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6579         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6580         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6581         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6582         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6583
6584         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6585                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6586                 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6587         }
6588
6589         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6590                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6591                 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6592         }
6593
6594         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6595                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6596                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6597         }
6598
6599         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6600                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6601                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6602         }
6603
6604         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6605         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6606         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6607         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6608 }
6609
6610 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6611                                           void *ras_error_status)
6612 {
6613         struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6614         uint32_t sec_count = 0, ded_count = 0;
6615         uint32_t i, j, k;
6616         uint32_t reg_value;
6617
6618         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6619                 return -EINVAL;
6620
6621         err_data->ue_count = 0;
6622         err_data->ce_count = 0;
6623
6624         mutex_lock(&adev->grbm_idx_mutex);
6625
6626         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6627                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6628                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6629                                 gfx_v9_0_select_se_sh(adev, j, 0, k);
6630                                 reg_value =
6631                                         RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6632                                 if (reg_value)
6633                                         gfx_v9_0_ras_error_count(adev,
6634                                                 &gfx_v9_0_edc_counter_regs[i],
6635                                                 j, k, reg_value,
6636                                                 &sec_count, &ded_count);
6637                         }
6638                 }
6639         }
6640
6641         err_data->ce_count += sec_count;
6642         err_data->ue_count += ded_count;
6643
6644         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6645         mutex_unlock(&adev->grbm_idx_mutex);
6646
6647         gfx_v9_0_query_utc_edc_status(adev, err_data);
6648
6649         return 0;
6650 }
6651
6652 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6653 {
6654         const unsigned int cp_coher_cntl =
6655                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6656                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6657                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6658                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6659                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6660
6661         /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
6662         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6663         amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
6664         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6665         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6666         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6667         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6668         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6669 }
6670
6671 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6672                                         uint32_t pipe, bool enable)
6673 {
6674         struct amdgpu_device *adev = ring->adev;
6675         uint32_t val;
6676         uint32_t wcl_cs_reg;
6677
6678         /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6679         val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
6680
6681         switch (pipe) {
6682         case 0:
6683                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
6684                 break;
6685         case 1:
6686                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
6687                 break;
6688         case 2:
6689                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
6690                 break;
6691         case 3:
6692                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
6693                 break;
6694         default:
6695                 DRM_DEBUG("invalid pipe %d\n", pipe);
6696                 return;
6697         }
6698
6699         amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6700
6701 }
6702 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6703 {
6704         struct amdgpu_device *adev = ring->adev;
6705         uint32_t val;
6706         int i;
6707
6708
6709         /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6710          * number of gfx waves. Setting 5 bit will make sure gfx only gets
6711          * around 25% of gpu resources.
6712          */
6713         val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6714         amdgpu_ring_emit_wreg(ring,
6715                               SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
6716                               val);
6717
6718         /* Restrict waves for normal/low priority compute queues as well
6719          * to get best QoS for high priority compute jobs.
6720          *
6721          * amdgpu controls only 1st ME(0-3 CS pipes).
6722          */
6723         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6724                 if (i != ring->pipe)
6725                         gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
6726
6727         }
6728 }
6729
6730 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6731         .name = "gfx_v9_0",
6732         .early_init = gfx_v9_0_early_init,
6733         .late_init = gfx_v9_0_late_init,
6734         .sw_init = gfx_v9_0_sw_init,
6735         .sw_fini = gfx_v9_0_sw_fini,
6736         .hw_init = gfx_v9_0_hw_init,
6737         .hw_fini = gfx_v9_0_hw_fini,
6738         .suspend = gfx_v9_0_suspend,
6739         .resume = gfx_v9_0_resume,
6740         .is_idle = gfx_v9_0_is_idle,
6741         .wait_for_idle = gfx_v9_0_wait_for_idle,
6742         .soft_reset = gfx_v9_0_soft_reset,
6743         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6744         .set_powergating_state = gfx_v9_0_set_powergating_state,
6745         .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6746 };
6747
6748 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6749         .type = AMDGPU_RING_TYPE_GFX,
6750         .align_mask = 0xff,
6751         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6752         .support_64bit_ptrs = true,
6753         .vmhub = AMDGPU_GFXHUB_0,
6754         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6755         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6756         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6757         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6758                 5 +  /* COND_EXEC */
6759                 7 +  /* PIPELINE_SYNC */
6760                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6761                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6762                 2 + /* VM_FLUSH */
6763                 8 +  /* FENCE for VM_FLUSH */
6764                 20 + /* GDS switch */
6765                 4 + /* double SWITCH_BUFFER,
6766                        the first COND_EXEC jump to the place just
6767                            prior to this double SWITCH_BUFFER  */
6768                 5 + /* COND_EXEC */
6769                 7 +      /*     HDP_flush */
6770                 4 +      /*     VGT_flush */
6771                 14 + /* CE_META */
6772                 31 + /* DE_META */
6773                 3 + /* CNTX_CTRL */
6774                 5 + /* HDP_INVL */
6775                 8 + 8 + /* FENCE x2 */
6776                 2 + /* SWITCH_BUFFER */
6777                 7, /* gfx_v9_0_emit_mem_sync */
6778         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6779         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6780         .emit_fence = gfx_v9_0_ring_emit_fence,
6781         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6782         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6783         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6784         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6785         .test_ring = gfx_v9_0_ring_test_ring,
6786         .test_ib = gfx_v9_0_ring_test_ib,
6787         .insert_nop = amdgpu_ring_insert_nop,
6788         .pad_ib = amdgpu_ring_generic_pad_ib,
6789         .emit_switch_buffer = gfx_v9_ring_emit_sb,
6790         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6791         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6792         .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6793         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6794         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6795         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6796         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6797         .soft_recovery = gfx_v9_0_ring_soft_recovery,
6798         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6799 };
6800
6801 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6802         .type = AMDGPU_RING_TYPE_COMPUTE,
6803         .align_mask = 0xff,
6804         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6805         .support_64bit_ptrs = true,
6806         .vmhub = AMDGPU_GFXHUB_0,
6807         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6808         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6809         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6810         .emit_frame_size =
6811                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6812                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6813                 5 + /* hdp invalidate */
6814                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6815                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6816                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6817                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6818                 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6819                 7 + /* gfx_v9_0_emit_mem_sync */
6820                 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6821                 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6822         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6823         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6824         .emit_fence = gfx_v9_0_ring_emit_fence,
6825         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6826         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6827         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6828         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6829         .test_ring = gfx_v9_0_ring_test_ring,
6830         .test_ib = gfx_v9_0_ring_test_ib,
6831         .insert_nop = amdgpu_ring_insert_nop,
6832         .pad_ib = amdgpu_ring_generic_pad_ib,
6833         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6834         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6835         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6836         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6837         .emit_wave_limit = gfx_v9_0_emit_wave_limit,
6838 };
6839
6840 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6841         .type = AMDGPU_RING_TYPE_KIQ,
6842         .align_mask = 0xff,
6843         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6844         .support_64bit_ptrs = true,
6845         .vmhub = AMDGPU_GFXHUB_0,
6846         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6847         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6848         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6849         .emit_frame_size =
6850                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6851                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6852                 5 + /* hdp invalidate */
6853                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6854                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6855                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6856                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6857                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6858         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6859         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6860         .test_ring = gfx_v9_0_ring_test_ring,
6861         .insert_nop = amdgpu_ring_insert_nop,
6862         .pad_ib = amdgpu_ring_generic_pad_ib,
6863         .emit_rreg = gfx_v9_0_ring_emit_rreg,
6864         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6865         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6866         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6867 };
6868
6869 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6870 {
6871         int i;
6872
6873         adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
6874
6875         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6876                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
6877
6878         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6879                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
6880 }
6881
6882 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
6883         .set = gfx_v9_0_set_eop_interrupt_state,
6884         .process = gfx_v9_0_eop_irq,
6885 };
6886
6887 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
6888         .set = gfx_v9_0_set_priv_reg_fault_state,
6889         .process = gfx_v9_0_priv_reg_irq,
6890 };
6891
6892 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
6893         .set = gfx_v9_0_set_priv_inst_fault_state,
6894         .process = gfx_v9_0_priv_inst_irq,
6895 };
6896
6897 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
6898         .set = gfx_v9_0_set_cp_ecc_error_state,
6899         .process = amdgpu_gfx_cp_ecc_error_irq,
6900 };
6901
6902
6903 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
6904 {
6905         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6906         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
6907
6908         adev->gfx.priv_reg_irq.num_types = 1;
6909         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
6910
6911         adev->gfx.priv_inst_irq.num_types = 1;
6912         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
6913
6914         adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
6915         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
6916 }
6917
6918 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
6919 {
6920         switch (adev->asic_type) {
6921         case CHIP_VEGA10:
6922         case CHIP_VEGA12:
6923         case CHIP_VEGA20:
6924         case CHIP_RAVEN:
6925         case CHIP_ARCTURUS:
6926         case CHIP_RENOIR:
6927                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
6928                 break;
6929         default:
6930                 break;
6931         }
6932 }
6933
6934 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
6935 {
6936         /* init asci gds info */
6937         switch (adev->asic_type) {
6938         case CHIP_VEGA10:
6939         case CHIP_VEGA12:
6940         case CHIP_VEGA20:
6941                 adev->gds.gds_size = 0x10000;
6942                 break;
6943         case CHIP_RAVEN:
6944         case CHIP_ARCTURUS:
6945                 adev->gds.gds_size = 0x1000;
6946                 break;
6947         default:
6948                 adev->gds.gds_size = 0x10000;
6949                 break;
6950         }
6951
6952         switch (adev->asic_type) {
6953         case CHIP_VEGA10:
6954         case CHIP_VEGA20:
6955                 adev->gds.gds_compute_max_wave_id = 0x7ff;
6956                 break;
6957         case CHIP_VEGA12:
6958                 adev->gds.gds_compute_max_wave_id = 0x27f;
6959                 break;
6960         case CHIP_RAVEN:
6961                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
6962                         adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
6963                 else
6964                         adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
6965                 break;
6966         case CHIP_ARCTURUS:
6967                 adev->gds.gds_compute_max_wave_id = 0xfff;
6968                 break;
6969         default:
6970                 /* this really depends on the chip */
6971                 adev->gds.gds_compute_max_wave_id = 0x7ff;
6972                 break;
6973         }
6974
6975         adev->gds.gws_size = 64;
6976         adev->gds.oa_size = 16;
6977 }
6978
6979 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
6980                                                  u32 bitmap)
6981 {
6982         u32 data;
6983
6984         if (!bitmap)
6985                 return;
6986
6987         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
6988         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
6989
6990         WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
6991 }
6992
6993 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
6994 {
6995         u32 data, mask;
6996
6997         data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
6998         data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
6999
7000         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7001         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7002
7003         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7004
7005         return (~data) & mask;
7006 }
7007
7008 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7009                                  struct amdgpu_cu_info *cu_info)
7010 {
7011         int i, j, k, counter, active_cu_number = 0;
7012         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7013         unsigned disable_masks[4 * 4];
7014
7015         if (!adev || !cu_info)
7016                 return -EINVAL;
7017
7018         /*
7019          * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7020          */
7021         if (adev->gfx.config.max_shader_engines *
7022                 adev->gfx.config.max_sh_per_se > 16)
7023                 return -EINVAL;
7024
7025         amdgpu_gfx_parse_disable_cu(disable_masks,
7026                                     adev->gfx.config.max_shader_engines,
7027                                     adev->gfx.config.max_sh_per_se);
7028
7029         mutex_lock(&adev->grbm_idx_mutex);
7030         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7031                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7032                         mask = 1;
7033                         ao_bitmap = 0;
7034                         counter = 0;
7035                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
7036                         gfx_v9_0_set_user_cu_inactive_bitmap(
7037                                 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7038                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7039
7040                         /*
7041                          * The bitmap(and ao_cu_bitmap) in cu_info structure is
7042                          * 4x4 size array, and it's usually suitable for Vega
7043                          * ASICs which has 4*2 SE/SH layout.
7044                          * But for Arcturus, SE/SH layout is changed to 8*1.
7045                          * To mostly reduce the impact, we make it compatible
7046                          * with current bitmap array as below:
7047                          *    SE4,SH0 --> bitmap[0][1]
7048                          *    SE5,SH0 --> bitmap[1][1]
7049                          *    SE6,SH0 --> bitmap[2][1]
7050                          *    SE7,SH0 --> bitmap[3][1]
7051                          */
7052                         cu_info->bitmap[i % 4][j + i / 4] = bitmap;
7053
7054                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7055                                 if (bitmap & mask) {
7056                                         if (counter < adev->gfx.config.max_cu_per_sh)
7057                                                 ao_bitmap |= mask;
7058                                         counter ++;
7059                                 }
7060                                 mask <<= 1;
7061                         }
7062                         active_cu_number += counter;
7063                         if (i < 2 && j < 2)
7064                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7065                         cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7066                 }
7067         }
7068         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7069         mutex_unlock(&adev->grbm_idx_mutex);
7070
7071         cu_info->number = active_cu_number;
7072         cu_info->ao_cu_mask = ao_cu_mask;
7073         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7074
7075         return 0;
7076 }
7077
7078 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7079 {
7080         .type = AMD_IP_BLOCK_TYPE_GFX,
7081         .major = 9,
7082         .minor = 0,
7083         .rev = 0,
7084         .funcs = &gfx_v9_0_ip_funcs,
7085 };