drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "soc15.h"
  33 #include "soc15d.h"
  34 #include "amdgpu_atomfirmware.h"
  35 #include "amdgpu_pm.h"
  36
  37 #include "gc/gc_9_0_offset.h"
  38 #include "gc/gc_9_0_sh_mask.h"
  39
  40 #include "vega10_enum.h"
  41 #include "hdp/hdp_4_0_offset.h"
  42
  43 #include "soc15_common.h"
  44 #include "clearstate_gfx9.h"
  45 #include "v9_structs.h"
  46
  47 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
  48
  49 #include "amdgpu_ras.h"
  50
  51 #include "gfx_v9_4.h"
  52
  53 #include "asic_reg/pwr/pwr_10_0_offset.h"
  54 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
  55
  56 #define GFX9_NUM_GFX_RINGS     1
  57 #define GFX9_MEC_HPD_SIZE 4096
  58 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
  59 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
  60
  61 #define mmGCEA_PROBE_MAP                        0x070c
  62 #define mmGCEA_PROBE_MAP_BASE_IDX               0
  63
  64 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  65 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  66 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  67 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  68 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  69 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  70
  71 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
  72 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
  73 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
  74 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
  75 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
  76 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
  77
  78 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
  79 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
  80 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
  81 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
  82 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
  83 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
  84
  85 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
  86 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
  87 MODULE_FIRMWARE("amdgpu/raven_me.bin");
  88 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
  89 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
  91
  92 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
  93 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
  94 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
  95 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
  96 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
  97 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
  98 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
  99
 100 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
 101 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
 102 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
 103 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
 104 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
 105 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
 106 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
 107
 108 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
 109 MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin");
 110 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
 111
 112 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
 113 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
 114 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
 115 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
 116 MODULE_FIRMWARE("amdgpu/renoir_mec2.bin");
 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
 118
 119 #define mmTCP_CHAN_STEER_0_ARCT                                                         0x0b03
 120 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX                                                        0
 121 #define mmTCP_CHAN_STEER_1_ARCT                                                         0x0b04
 122 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX                                                        0
 123 #define mmTCP_CHAN_STEER_2_ARCT                                                         0x0b09
 124 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX                                                        0
 125 #define mmTCP_CHAN_STEER_3_ARCT                                                         0x0b0a
 126 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX                                                        0
 127 #define mmTCP_CHAN_STEER_4_ARCT                                                         0x0b0b
 128 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX                                                        0
 129 #define mmTCP_CHAN_STEER_5_ARCT                                                         0x0b0c
 130 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX                                                        0
 131
 132 enum ta_ras_gfx_subblock {
 133         /*CPC*/
 134         TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
 135         TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
 136         TA_RAS_BLOCK__GFX_CPC_UCODE,
 137         TA_RAS_BLOCK__GFX_DC_STATE_ME1,
 138         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
 139         TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
 140         TA_RAS_BLOCK__GFX_DC_STATE_ME2,
 141         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
 142         TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 143         TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 144         /* CPF*/
 145         TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 146         TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 147         TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
 148         TA_RAS_BLOCK__GFX_CPF_TAG,
 149         TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
 150         /* CPG*/
 151         TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 152         TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 153         TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
 154         TA_RAS_BLOCK__GFX_CPG_TAG,
 155         TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
 156         /* GDS*/
 157         TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 158         TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 159         TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
 160         TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
 161         TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
 162         TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 163         TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 164         /* SPI*/
 165         TA_RAS_BLOCK__GFX_SPI_SR_MEM,
 166         /* SQ*/
 167         TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 168         TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 169         TA_RAS_BLOCK__GFX_SQ_LDS_D,
 170         TA_RAS_BLOCK__GFX_SQ_LDS_I,
 171         TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
 172         TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
 173         /* SQC (3 ranges)*/
 174         TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 175         /* SQC range 0*/
 176         TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 177         TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
 178                 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
 179         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
 180         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
 181         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
 182         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
 183         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
 184         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 185         TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
 186                 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 187         /* SQC range 1*/
 188         TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 189         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
 190                 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 191         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
 192         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
 193         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
 194         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
 195         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
 196         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
 197         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
 198         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 199         TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
 200                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 201         /* SQC range 2*/
 202         TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 203         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
 204                 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 205         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
 206         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
 207         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
 208         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
 209         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
 210         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
 211         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
 212         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 213         TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
 214                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 215         TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
 216         /* TA*/
 217         TA_RAS_BLOCK__GFX_TA_INDEX_START,
 218         TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
 219         TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
 220         TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
 221         TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
 222         TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 223         TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 224         /* TCA*/
 225         TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 226         TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 227         TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 228         TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 229         /* TCC (5 sub-ranges)*/
 230         TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 231         /* TCC range 0*/
 232         TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 233         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
 234         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
 235         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
 236         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
 237         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
 238         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
 239         TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
 240         TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 241         TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 242         /* TCC range 1*/
 243         TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 244         TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 245         TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 246         TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
 247                 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 248         /* TCC range 2*/
 249         TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 250         TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 251         TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
 252         TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
 253         TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
 254         TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
 255         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
 256         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
 257         TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 258         TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
 259                 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 260         /* TCC range 3*/
 261         TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 262         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 263         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 264         TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
 265                 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 266         /* TCC range 4*/
 267         TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 268         TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
 269                 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 270         TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 271         TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
 272                 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 273         TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
 274         /* TCI*/
 275         TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
 276         /* TCP*/
 277         TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 278         TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 279         TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
 280         TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
 281         TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
 282         TA_RAS_BLOCK__GFX_TCP_DB_RAM,
 283         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
 284         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 285         TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 286         /* TD*/
 287         TA_RAS_BLOCK__GFX_TD_INDEX_START,
 288         TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
 289         TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
 290         TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 291         TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 292         /* EA (3 sub-ranges)*/
 293         TA_RAS_BLOCK__GFX_EA_INDEX_START,
 294         /* EA range 0*/
 295         TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
 296         TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
 297         TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
 298         TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
 299         TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
 300         TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
 301         TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
 302         TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
 303         TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 304         TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 305         /* EA range 1*/
 306         TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 307         TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 308         TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
 309         TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
 310         TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
 311         TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
 312         TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
 313         TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 314         TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 315         /* EA range 2*/
 316         TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 317         TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 318         TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
 319         TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
 320         TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 321         TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 322         TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
 323         /* UTC VM L2 bank*/
 324         TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
 325         /* UTC VM walker*/
 326         TA_RAS_BLOCK__UTC_VML2_WALKER,
 327         /* UTC ATC L2 2MB cache*/
 328         TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
 329         /* UTC ATC L2 4KB cache*/
 330         TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
 331         TA_RAS_BLOCK__GFX_MAX
 332 };
 333
 334 struct ras_gfx_subblock {
 335         unsigned char *name;
 336         int ta_subblock;
 337         int hw_supported_error_type;
 338         int sw_supported_error_type;
 339 };
 340
 341 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \
 342         [AMDGPU_RAS_BLOCK__##subblock] = {                                     \
 343                 #subblock,                                                     \
 344                 TA_RAS_BLOCK__##subblock,                                      \
 345                 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \
 346                 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \
 347         }
 348
 349 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
 350         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
 351         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
 352         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 353         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 354         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 355         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 356         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 357         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 358         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 359         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 360         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
 361         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
 362         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
 363         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
 364         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 365         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
 366         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
 367                              0),
 368         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
 369                              0),
 370         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 371         AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 372         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 373         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
 374         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
 375         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 376         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
 377         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 378                              0, 0),
 379         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 380                              0),
 381         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 382                              0, 0),
 383         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
 384                              0),
 385         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 386                              0, 0),
 387         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 388                              0),
 389         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 390                              1),
 391         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 392                              0, 0, 0),
 393         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 394                              0),
 395         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 396                              0),
 397         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 398                              0),
 399         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 400                              0),
 401         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 402                              0),
 403         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 404                              0, 0),
 405         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 406                              0),
 407         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 408                              0),
 409         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 410                              0, 0, 0),
 411         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 412                              0),
 413         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 414                              0),
 415         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 416                              0),
 417         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 418                              0),
 419         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 420                              0),
 421         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 422                              0, 0),
 423         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 424                              0),
 425         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
 426         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 427         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 428         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 429         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 430         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
 431         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 432         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
 433         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
 434                              1),
 435         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
 436                              1),
 437         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
 438                              1),
 439         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
 440                              0),
 441         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
 442                              0),
 443         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 444         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 445         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
 446         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
 447         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
 448         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
 449         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 450         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
 451         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
 452         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 453         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
 454         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
 455                              0),
 456         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 457         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
 458                              0),
 459         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
 460                              0, 0),
 461         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
 462                              0),
 463         AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 464         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
 465         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
 466         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 467         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 468         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 469         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
 470         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
 471         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
 472         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
 473         AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 474         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
 475         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 476         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 477         AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 478         AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 479         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 480         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 481         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 482         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 483         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 484         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 485         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 486         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 487         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 488         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 489         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 490         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 491         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 492         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 493         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
 494         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
 495         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
 496         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
 497 };
 498
 499 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
 500 {
 501         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 502         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
 503         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 504         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 505         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 506         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 507         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 508         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 509         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 510         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
 511         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
 512         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 513         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 514         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 515         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 516         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 517         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
 518         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 519         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 520         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 521 };
 522
 523 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 524 {
 525         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 526         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 527         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 528         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 529         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 530         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 531         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
 532         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 533         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
 534         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 535         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 536         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 537         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 538         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 539         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 540         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
 541         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
 542         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 543 };
 544
 545 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
 546 {
 547         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
 548         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 549         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 550         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
 551         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
 552         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
 553         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
 554         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
 555         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
 556         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
 557         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
 558 };
 559
 560 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 561 {
 562         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 563         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 564         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 565         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 566         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 567         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 568         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 569         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 570         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 571         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 572         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 573         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 574         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 575         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 576         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 577         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 578         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 579         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 580         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 581         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
 582         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
 583         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 584         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 585         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 586 };
 587
 588 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 589 {
 590         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 591         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
 592         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
 593         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
 594         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
 595         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 596         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 597 };
 598
 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
 600 {
 601         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
 602         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 603         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 604         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
 605         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
 606         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
 607         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
 608         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
 609         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 610         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 611         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
 612         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
 613         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
 614         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 615         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 616         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 617         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
 618         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 619         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 620 };
 621
 622 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
 623 {
 624         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 625         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 626         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 627         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
 628         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
 629         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 630         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 631         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 632         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 633         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 634         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 635         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
 636 };
 637
 638 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 639 {
 640         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
 641         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
 642         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 643 };
 644
 645 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
 646 {
 647         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 648         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 649         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 650         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 651         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 652         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 653         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 654         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 655         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
 656         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 657         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 658         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 659         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 660         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 661         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 662         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 663 };
 664
 665 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 666 {
 667         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
 668         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 669         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 670         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
 671         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
 672         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 673         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
 674         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 675         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
 676         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 677         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 678         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 679         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 680 };
 681
 682 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
 683 {
 684         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 685         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
 686         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
 687         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
 688         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
 689         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
 690         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
 691         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
 692         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
 693         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
 694 };
 695
 696 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
 697         {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
 698         {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
 699 };
 700
 701 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
 702 {
 703         mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 704         mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 705         mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 706         mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 707         mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 708         mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 709         mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 710         mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 711 };
 712
 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 714 {
 715         mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 716         mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 717         mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 718         mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 719         mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 720         mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 721         mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 722         mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 723 };
 724
 725 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v)
 726 {
 727         static void *scratch_reg0;
 728         static void *scratch_reg1;
 729         static void *scratch_reg2;
 730         static void *scratch_reg3;
 731         static void *spare_int;
 732         static uint32_t grbm_cntl;
 733         static uint32_t grbm_idx;
 734
 735         scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4;
 736         scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4;
 737         scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4;
 738         scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4;
 739         spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4;
 740
 741         grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;
 742         grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;
 743
 744         if (amdgpu_sriov_runtime(adev)) {
 745                 pr_err("shouldn't call rlcg write register during runtime\n");
 746                 return;
 747         }
 748
 749         if (offset == grbm_cntl || offset == grbm_idx) {
 750                 if (offset  == grbm_cntl)
 751                         writel(v, scratch_reg2);
 752                 else if (offset == grbm_idx)
 753                         writel(v, scratch_reg3);
 754
 755                 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
 756         } else {
 757                 uint32_t i = 0;
 758                 uint32_t retries = 50000;
 759
 760                 writel(v, scratch_reg0);
 761                 writel(offset | 0x80000000, scratch_reg1);
 762                 writel(1, spare_int);
 763                 for (i = 0; i < retries; i++) {
 764                         u32 tmp;
 765
 766                         tmp = readl(scratch_reg1);
 767                         if (!(tmp & 0x80000000))
 768                                 break;
 769
 770                         udelay(10);
 771                 }
 772                 if (i >= retries)
 773                         pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
 774         }
 775
 776 }
 777
 778 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 779 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 780 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 781 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
 782
 783 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 784 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 785 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 786 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 787 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 788                                  struct amdgpu_cu_info *cu_info);
 789 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 790 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
 791 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
 792 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
 793 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
 794                                           void *ras_error_status);
 795 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
 796                                      void *inject_if);
 797 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
 798
 799 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 800                                 uint64_t queue_mask)
 801 {
 802         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 803         amdgpu_ring_write(kiq_ring,
 804                 PACKET3_SET_RESOURCES_VMID_MASK(0) |
 805                 /* vmid_mask:0* queue_type:0 (KIQ) */
 806                 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
 807         amdgpu_ring_write(kiq_ring,
 808                         lower_32_bits(queue_mask));     /* queue mask lo */
 809         amdgpu_ring_write(kiq_ring,
 810                         upper_32_bits(queue_mask));     /* queue mask hi */
 811         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 812         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 813         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 814         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 815 }
 816
 817 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 818                                  struct amdgpu_ring *ring)
 819 {
 820         struct amdgpu_device *adev = kiq_ring->adev;
 821         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 822         uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
 823         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 824
 825         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 826         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 827         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 828                          PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 829                          PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 830                          PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 831                          PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 832                          PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
 833                          /*queue_type: normal compute queue */
 834                          PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
 835                          /* alloc format: all_on_one_pipe */
 836                          PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
 837                          PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 838                          /* num_queues: must be 1 */
 839                          PACKET3_MAP_QUEUES_NUM_QUEUES(1));
 840         amdgpu_ring_write(kiq_ring,
 841                         PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 842         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 843         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 844         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 845         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 846 }
 847
 848 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 849                                    struct amdgpu_ring *ring,
 850                                    enum amdgpu_unmap_queues_action action,
 851                                    u64 gpu_addr, u64 seq)
 852 {
 853         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 854
 855         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 856         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 857                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 858                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 859                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 860                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 861         amdgpu_ring_write(kiq_ring,
 862                         PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 863
 864         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 865                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 866                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 867                 amdgpu_ring_write(kiq_ring, seq);
 868         } else {
 869                 amdgpu_ring_write(kiq_ring, 0);
 870                 amdgpu_ring_write(kiq_ring, 0);
 871                 amdgpu_ring_write(kiq_ring, 0);
 872         }
 873 }
 874
 875 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 876                                    struct amdgpu_ring *ring,
 877                                    u64 addr,
 878                                    u64 seq)
 879 {
 880         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 881
 882         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 883         amdgpu_ring_write(kiq_ring,
 884                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 885                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 886                           PACKET3_QUERY_STATUS_COMMAND(2));
 887         /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 888         amdgpu_ring_write(kiq_ring,
 889                         PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 890                         PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 891         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 892         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 893         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 894         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 895 }
 896
 897 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 898                                 uint16_t pasid, uint32_t flush_type,
 899                                 bool all_hub)
 900 {
 901         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
 902         amdgpu_ring_write(kiq_ring,
 903                         PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
 904                         PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
 905                         PACKET3_INVALIDATE_TLBS_PASID(pasid) |
 906                         PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
 907 }
 908
 909 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
 910         .kiq_set_resources = gfx_v9_0_kiq_set_resources,
 911         .kiq_map_queues = gfx_v9_0_kiq_map_queues,
 912         .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
 913         .kiq_query_status = gfx_v9_0_kiq_query_status,
 914         .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
 915         .set_resources_size = 8,
 916         .map_queues_size = 7,
 917         .unmap_queues_size = 6,
 918         .query_status_size = 7,
 919         .invalidate_tlbs_size = 2,
 920 };
 921
 922 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 923 {
 924         adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
 925 }
 926
 927 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 928 {
 929         switch (adev->asic_type) {
 930         case CHIP_VEGA10:
 931                 soc15_program_register_sequence(adev,
 932                                                 golden_settings_gc_9_0,
 933                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 934                 soc15_program_register_sequence(adev,
 935                                                 golden_settings_gc_9_0_vg10,
 936                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
 937                 break;
 938         case CHIP_VEGA12:
 939                 soc15_program_register_sequence(adev,
 940                                                 golden_settings_gc_9_2_1,
 941                                                 ARRAY_SIZE(golden_settings_gc_9_2_1));
 942                 soc15_program_register_sequence(adev,
 943                                                 golden_settings_gc_9_2_1_vg12,
 944                                                 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
 945                 break;
 946         case CHIP_VEGA20:
 947                 soc15_program_register_sequence(adev,
 948                                                 golden_settings_gc_9_0,
 949                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 950                 soc15_program_register_sequence(adev,
 951                                                 golden_settings_gc_9_0_vg20,
 952                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
 953                 break;
 954         case CHIP_ARCTURUS:
 955                 soc15_program_register_sequence(adev,
 956                                                 golden_settings_gc_9_4_1_arct,
 957                                                 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
 958                 break;
 959         case CHIP_RAVEN:
 960                 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
 961                                                 ARRAY_SIZE(golden_settings_gc_9_1));
 962                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
 963                         soc15_program_register_sequence(adev,
 964                                                         golden_settings_gc_9_1_rv2,
 965                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv2));
 966                 else
 967                         soc15_program_register_sequence(adev,
 968                                                         golden_settings_gc_9_1_rv1,
 969                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv1));
 970                 break;
 971          case CHIP_RENOIR:
 972                 soc15_program_register_sequence(adev,
 973                                                 golden_settings_gc_9_1_rn,
 974                                                 ARRAY_SIZE(golden_settings_gc_9_1_rn));
 975                 return; /* for renoir, don't need common goldensetting */
 976         default:
 977                 break;
 978         }
 979
 980         if (adev->asic_type != CHIP_ARCTURUS)
 981                 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
 982                                                 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
 983 }
 984
 985 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
 986 {
 987         adev->gfx.scratch.num_reg = 8;
 988         adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
 989         adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 990 }
 991
 992 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
 993                                        bool wc, uint32_t reg, uint32_t val)
 994 {
 995         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 996         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
 997                                 WRITE_DATA_DST_SEL(0) |
 998                                 (wc ? WR_CONFIRM : 0));
 999         amdgpu_ring_write(ring, reg);
1000         amdgpu_ring_write(ring, 0);
1001         amdgpu_ring_write(ring, val);
1002 }
1003
1004 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1005                                   int mem_space, int opt, uint32_t addr0,
1006                                   uint32_t addr1, uint32_t ref, uint32_t mask,
1007                                   uint32_t inv)
1008 {
1009         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1010         amdgpu_ring_write(ring,
1011                                  /* memory (1) or register (0) */
1012                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1013                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
1014                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
1015                                  WAIT_REG_MEM_ENGINE(eng_sel)));
1016
1017         if (mem_space)
1018                 BUG_ON(addr0 & 0x3); /* Dword align */
1019         amdgpu_ring_write(ring, addr0);
1020         amdgpu_ring_write(ring, addr1);
1021         amdgpu_ring_write(ring, ref);
1022         amdgpu_ring_write(ring, mask);
1023         amdgpu_ring_write(ring, inv); /* poll interval */
1024 }
1025
1026 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1027 {
1028         struct amdgpu_device *adev = ring->adev;
1029         uint32_t scratch;
1030         uint32_t tmp = 0;
1031         unsigned i;
1032         int r;
1033
1034         r = amdgpu_gfx_scratch_get(adev, &scratch);
1035         if (r)
1036                 return r;
1037
1038         WREG32(scratch, 0xCAFEDEAD);
1039         r = amdgpu_ring_alloc(ring, 3);
1040         if (r)
1041                 goto error_free_scratch;
1042
1043         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1044         amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
1045         amdgpu_ring_write(ring, 0xDEADBEEF);
1046         amdgpu_ring_commit(ring);
1047
1048         for (i = 0; i < adev->usec_timeout; i++) {
1049                 tmp = RREG32(scratch);
1050                 if (tmp == 0xDEADBEEF)
1051                         break;
1052                 udelay(1);
1053         }
1054
1055         if (i >= adev->usec_timeout)
1056                 r = -ETIMEDOUT;
1057
1058 error_free_scratch:
1059         amdgpu_gfx_scratch_free(adev, scratch);
1060         return r;
1061 }
1062
1063 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1064 {
1065         struct amdgpu_device *adev = ring->adev;
1066         struct amdgpu_ib ib;
1067         struct dma_fence *f = NULL;
1068
1069         unsigned index;
1070         uint64_t gpu_addr;
1071         uint32_t tmp;
1072         long r;
1073
1074         r = amdgpu_device_wb_get(adev, &index);
1075         if (r)
1076                 return r;
1077
1078         gpu_addr = adev->wb.gpu_addr + (index * 4);
1079         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1080         memset(&ib, 0, sizeof(ib));
1081         r = amdgpu_ib_get(adev, NULL, 16,
1082                                         AMDGPU_IB_POOL_DIRECT, &ib);
1083         if (r)
1084                 goto err1;
1085
1086         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1087         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1088         ib.ptr[2] = lower_32_bits(gpu_addr);
1089         ib.ptr[3] = upper_32_bits(gpu_addr);
1090         ib.ptr[4] = 0xDEADBEEF;
1091         ib.length_dw = 5;
1092
1093         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1094         if (r)
1095                 goto err2;
1096
1097         r = dma_fence_wait_timeout(f, false, timeout);
1098         if (r == 0) {
1099                 r = -ETIMEDOUT;
1100                 goto err2;
1101         } else if (r < 0) {
1102                 goto err2;
1103         }
1104
1105         tmp = adev->wb.wb[index];
1106         if (tmp == 0xDEADBEEF)
1107                 r = 0;
1108         else
1109                 r = -EINVAL;
1110
1111 err2:
1112         amdgpu_ib_free(adev, &ib, NULL);
1113         dma_fence_put(f);
1114 err1:
1115         amdgpu_device_wb_free(adev, index);
1116         return r;
1117 }
1118
1119
1120 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1121 {
1122         release_firmware(adev->gfx.pfp_fw);
1123         adev->gfx.pfp_fw = NULL;
1124         release_firmware(adev->gfx.me_fw);
1125         adev->gfx.me_fw = NULL;
1126         release_firmware(adev->gfx.ce_fw);
1127         adev->gfx.ce_fw = NULL;
1128         release_firmware(adev->gfx.rlc_fw);
1129         adev->gfx.rlc_fw = NULL;
1130         release_firmware(adev->gfx.mec_fw);
1131         adev->gfx.mec_fw = NULL;
1132         release_firmware(adev->gfx.mec2_fw);
1133         adev->gfx.mec2_fw = NULL;
1134
1135         kfree(adev->gfx.rlc.register_list_format);
1136 }
1137
1138 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
1139 {
1140         const struct rlc_firmware_header_v2_1 *rlc_hdr;
1141
1142         rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1143         adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
1144         adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
1145         adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
1146         adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
1147         adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
1148         adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
1149         adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
1150         adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
1151         adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
1152         adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
1153         adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
1154         adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
1155         adev->gfx.rlc.reg_list_format_direct_reg_list_length =
1156                         le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
1157 }
1158
1159 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1160 {
1161         adev->gfx.me_fw_write_wait = false;
1162         adev->gfx.mec_fw_write_wait = false;
1163
1164         if ((adev->asic_type != CHIP_ARCTURUS) &&
1165             ((adev->gfx.mec_fw_version < 0x000001a5) ||
1166             (adev->gfx.mec_feature_version < 46) ||
1167             (adev->gfx.pfp_fw_version < 0x000000b7) ||
1168             (adev->gfx.pfp_feature_version < 46)))
1169                 DRM_WARN_ONCE("CP firmware version too old, please update!");
1170
1171         switch (adev->asic_type) {
1172         case CHIP_VEGA10:
1173                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1174                     (adev->gfx.me_feature_version >= 42) &&
1175                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1176                     (adev->gfx.pfp_feature_version >= 42))
1177                         adev->gfx.me_fw_write_wait = true;
1178
1179                 if ((adev->gfx.mec_fw_version >=  0x00000193) &&
1180                     (adev->gfx.mec_feature_version >= 42))
1181                         adev->gfx.mec_fw_write_wait = true;
1182                 break;
1183         case CHIP_VEGA12:
1184                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1185                     (adev->gfx.me_feature_version >= 44) &&
1186                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1187                     (adev->gfx.pfp_feature_version >= 44))
1188                         adev->gfx.me_fw_write_wait = true;
1189
1190                 if ((adev->gfx.mec_fw_version >=  0x00000196) &&
1191                     (adev->gfx.mec_feature_version >= 44))
1192                         adev->gfx.mec_fw_write_wait = true;
1193                 break;
1194         case CHIP_VEGA20:
1195                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1196                     (adev->gfx.me_feature_version >= 44) &&
1197                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1198                     (adev->gfx.pfp_feature_version >= 44))
1199                         adev->gfx.me_fw_write_wait = true;
1200
1201                 if ((adev->gfx.mec_fw_version >=  0x00000197) &&
1202                     (adev->gfx.mec_feature_version >= 44))
1203                         adev->gfx.mec_fw_write_wait = true;
1204                 break;
1205         case CHIP_RAVEN:
1206                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1207                     (adev->gfx.me_feature_version >= 42) &&
1208                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1209                     (adev->gfx.pfp_feature_version >= 42))
1210                         adev->gfx.me_fw_write_wait = true;
1211
1212                 if ((adev->gfx.mec_fw_version >=  0x00000192) &&
1213                     (adev->gfx.mec_feature_version >= 42))
1214                         adev->gfx.mec_fw_write_wait = true;
1215                 break;
1216         default:
1217                 adev->gfx.me_fw_write_wait = true;
1218                 adev->gfx.mec_fw_write_wait = true;
1219                 break;
1220         }
1221 }
1222
1223 struct amdgpu_gfxoff_quirk {
1224         u16 chip_vendor;
1225         u16 chip_device;
1226         u16 subsys_vendor;
1227         u16 subsys_device;
1228         u8 revision;
1229 };
1230
1231 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1232         /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1233         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1234         /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1235         { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1236         /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1237         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1238         { 0, 0, 0, 0, 0 },
1239 };
1240
1241 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1242 {
1243         const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1244
1245         while (p && p->chip_device != 0) {
1246                 if (pdev->vendor == p->chip_vendor &&
1247                     pdev->device == p->chip_device &&
1248                     pdev->subsystem_vendor == p->subsys_vendor &&
1249                     pdev->subsystem_device == p->subsys_device &&
1250                     pdev->revision == p->revision) {
1251                         return true;
1252                 }
1253                 ++p;
1254         }
1255         return false;
1256 }
1257
1258 static bool is_raven_kicker(struct amdgpu_device *adev)
1259 {
1260         if (adev->pm.fw_version >= 0x41e2b)
1261                 return true;
1262         else
1263                 return false;
1264 }
1265
1266 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1267 {
1268         if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1269                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1270
1271         switch (adev->asic_type) {
1272         case CHIP_VEGA10:
1273         case CHIP_VEGA12:
1274         case CHIP_VEGA20:
1275                 break;
1276         case CHIP_RAVEN:
1277                 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1278                       (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1279                     ((!is_raven_kicker(adev) &&
1280                       adev->gfx.rlc_fw_version < 531) ||
1281                      (adev->gfx.rlc_feature_version < 1) ||
1282                      !adev->gfx.rlc.is_rlc_v2_1))
1283                         adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1284
1285                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1286                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1287                                 AMD_PG_SUPPORT_CP |
1288                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1289                 break;
1290         case CHIP_RENOIR:
1291                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1292                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1293                                 AMD_PG_SUPPORT_CP |
1294                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1295                 break;
1296         default:
1297                 break;
1298         }
1299 }
1300
1301 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1302                                           const char *chip_name)
1303 {
1304         char fw_name[30];
1305         int err;
1306         struct amdgpu_firmware_info *info = NULL;
1307         const struct common_firmware_header *header = NULL;
1308         const struct gfx_firmware_header_v1_0 *cp_hdr;
1309
1310         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1311         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1312         if (err)
1313                 goto out;
1314         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1315         if (err)
1316                 goto out;
1317         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1318         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1319         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1320
1321         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1322         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1323         if (err)
1324                 goto out;
1325         err = amdgpu_ucode_validate(adev->gfx.me_fw);
1326         if (err)
1327                 goto out;
1328         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1329         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1330         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1331
1332         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1333         err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1334         if (err)
1335                 goto out;
1336         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1337         if (err)
1338                 goto out;
1339         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1340         adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1341         adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1342
1343         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1344                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1345                 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1346                 info->fw = adev->gfx.pfp_fw;
1347                 header = (const struct common_firmware_header *)info->fw->data;
1348                 adev->firmware.fw_size +=
1349                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1350
1351                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1352                 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1353                 info->fw = adev->gfx.me_fw;
1354                 header = (const struct common_firmware_header *)info->fw->data;
1355                 adev->firmware.fw_size +=
1356                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1357
1358                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1359                 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1360                 info->fw = adev->gfx.ce_fw;
1361                 header = (const struct common_firmware_header *)info->fw->data;
1362                 adev->firmware.fw_size +=
1363                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1364         }
1365
1366 out:
1367         if (err) {
1368                 dev_err(adev->dev,
1369                         "gfx9: Failed to load firmware \"%s\"\n",
1370                         fw_name);
1371                 release_firmware(adev->gfx.pfp_fw);
1372                 adev->gfx.pfp_fw = NULL;
1373                 release_firmware(adev->gfx.me_fw);
1374                 adev->gfx.me_fw = NULL;
1375                 release_firmware(adev->gfx.ce_fw);
1376                 adev->gfx.ce_fw = NULL;
1377         }
1378         return err;
1379 }
1380
1381 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1382                                           const char *chip_name)
1383 {
1384         char fw_name[30];
1385         int err;
1386         struct amdgpu_firmware_info *info = NULL;
1387         const struct common_firmware_header *header = NULL;
1388         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1389         unsigned int *tmp = NULL;
1390         unsigned int i = 0;
1391         uint16_t version_major;
1392         uint16_t version_minor;
1393         uint32_t smu_version;
1394
1395         /*
1396          * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1397          * instead of picasso_rlc.bin.
1398          * Judgment method:
1399          * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1400          *          or revision >= 0xD8 && revision <= 0xDF
1401          * otherwise is PCO FP5
1402          */
1403         if (!strcmp(chip_name, "picasso") &&
1404                 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1405                 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1406                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1407         else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1408                 (smu_version >= 0x41e2b))
1409                 /**
1410                 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1411                 */
1412                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1413         else
1414                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1415         err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1416         if (err)
1417                 goto out;
1418         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1419         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1420
1421         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1422         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1423         if (version_major == 2 && version_minor == 1)
1424                 adev->gfx.rlc.is_rlc_v2_1 = true;
1425
1426         adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1427         adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1428         adev->gfx.rlc.save_and_restore_offset =
1429                         le32_to_cpu(rlc_hdr->save_and_restore_offset);
1430         adev->gfx.rlc.clear_state_descriptor_offset =
1431                         le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1432         adev->gfx.rlc.avail_scratch_ram_locations =
1433                         le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1434         adev->gfx.rlc.reg_restore_list_size =
1435                         le32_to_cpu(rlc_hdr->reg_restore_list_size);
1436         adev->gfx.rlc.reg_list_format_start =
1437                         le32_to_cpu(rlc_hdr->reg_list_format_start);
1438         adev->gfx.rlc.reg_list_format_separate_start =
1439                         le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1440         adev->gfx.rlc.starting_offsets_start =
1441                         le32_to_cpu(rlc_hdr->starting_offsets_start);
1442         adev->gfx.rlc.reg_list_format_size_bytes =
1443                         le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1444         adev->gfx.rlc.reg_list_size_bytes =
1445                         le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1446         adev->gfx.rlc.register_list_format =
1447                         kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1448                                 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1449         if (!adev->gfx.rlc.register_list_format) {
1450                 err = -ENOMEM;
1451                 goto out;
1452         }
1453
1454         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1455                         le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1456         for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1457                 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1458
1459         adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1460
1461         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1462                         le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1463         for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1464                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1465
1466         if (adev->gfx.rlc.is_rlc_v2_1)
1467                 gfx_v9_0_init_rlc_ext_microcode(adev);
1468
1469         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1470                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1471                 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1472                 info->fw = adev->gfx.rlc_fw;
1473                 header = (const struct common_firmware_header *)info->fw->data;
1474                 adev->firmware.fw_size +=
1475                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1476
1477                 if (adev->gfx.rlc.is_rlc_v2_1 &&
1478                     adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
1479                     adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
1480                     adev->gfx.rlc.save_restore_list_srm_size_bytes) {
1481                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
1482                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
1483                         info->fw = adev->gfx.rlc_fw;
1484                         adev->firmware.fw_size +=
1485                                 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
1486
1487                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
1488                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
1489                         info->fw = adev->gfx.rlc_fw;
1490                         adev->firmware.fw_size +=
1491                                 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
1492
1493                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
1494                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
1495                         info->fw = adev->gfx.rlc_fw;
1496                         adev->firmware.fw_size +=
1497                                 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
1498                 }
1499         }
1500
1501 out:
1502         if (err) {
1503                 dev_err(adev->dev,
1504                         "gfx9: Failed to load firmware \"%s\"\n",
1505                         fw_name);
1506                 release_firmware(adev->gfx.rlc_fw);
1507                 adev->gfx.rlc_fw = NULL;
1508         }
1509         return err;
1510 }
1511
1512 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1513                                           const char *chip_name)
1514 {
1515         char fw_name[30];
1516         int err;
1517         struct amdgpu_firmware_info *info = NULL;
1518         const struct common_firmware_header *header = NULL;
1519         const struct gfx_firmware_header_v1_0 *cp_hdr;
1520
1521         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1522         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1523         if (err)
1524                 goto out;
1525         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1526         if (err)
1527                 goto out;
1528         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1529         adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1530         adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1531
1532
1533         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1534         err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1535         if (!err) {
1536                 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1537                 if (err)
1538                         goto out;
1539                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1540                 adev->gfx.mec2_fw->data;
1541                 adev->gfx.mec2_fw_version =
1542                 le32_to_cpu(cp_hdr->header.ucode_version);
1543                 adev->gfx.mec2_feature_version =
1544                 le32_to_cpu(cp_hdr->ucode_feature_version);
1545         } else {
1546                 err = 0;
1547                 adev->gfx.mec2_fw = NULL;
1548         }
1549
1550         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1551                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1552                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1553                 info->fw = adev->gfx.mec_fw;
1554                 header = (const struct common_firmware_header *)info->fw->data;
1555                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1556                 adev->firmware.fw_size +=
1557                         ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1558
1559                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
1560                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
1561                 info->fw = adev->gfx.mec_fw;
1562                 adev->firmware.fw_size +=
1563                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1564
1565                 if (adev->gfx.mec2_fw) {
1566                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1567                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1568                         info->fw = adev->gfx.mec2_fw;
1569                         header = (const struct common_firmware_header *)info->fw->data;
1570                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1571                         adev->firmware.fw_size +=
1572                                 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1573
1574                         /* TODO: Determine if MEC2 JT FW loading can be removed
1575                                  for all GFX V9 asic and above */
1576                         if (adev->asic_type != CHIP_ARCTURUS &&
1577                             adev->asic_type != CHIP_RENOIR) {
1578                                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
1579                                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
1580                                 info->fw = adev->gfx.mec2_fw;
1581                                 adev->firmware.fw_size +=
1582                                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
1583                                         PAGE_SIZE);
1584                         }
1585                 }
1586         }
1587
1588 out:
1589         gfx_v9_0_check_if_need_gfxoff(adev);
1590         gfx_v9_0_check_fw_write_wait(adev);
1591         if (err) {
1592                 dev_err(adev->dev,
1593                         "gfx9: Failed to load firmware \"%s\"\n",
1594                         fw_name);
1595                 release_firmware(adev->gfx.mec_fw);
1596                 adev->gfx.mec_fw = NULL;
1597                 release_firmware(adev->gfx.mec2_fw);
1598                 adev->gfx.mec2_fw = NULL;
1599         }
1600         return err;
1601 }
1602
1603 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1604 {
1605         const char *chip_name;
1606         int r;
1607
1608         DRM_DEBUG("\n");
1609
1610         switch (adev->asic_type) {
1611         case CHIP_VEGA10:
1612                 chip_name = "vega10";
1613                 break;
1614         case CHIP_VEGA12:
1615                 chip_name = "vega12";
1616                 break;
1617         case CHIP_VEGA20:
1618                 chip_name = "vega20";
1619                 break;
1620         case CHIP_RAVEN:
1621                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1622                         chip_name = "raven2";
1623                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1624                         chip_name = "picasso";
1625                 else
1626                         chip_name = "raven";
1627                 break;
1628         case CHIP_ARCTURUS:
1629                 chip_name = "arcturus";
1630                 break;
1631         case CHIP_RENOIR:
1632                 chip_name = "renoir";
1633                 break;
1634         default:
1635                 BUG();
1636         }
1637
1638         /* No CPG in Arcturus */
1639         if (adev->asic_type != CHIP_ARCTURUS) {
1640                 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name);
1641                 if (r)
1642                         return r;
1643         }
1644
1645         r = gfx_v9_0_init_rlc_microcode(adev, chip_name);
1646         if (r)
1647                 return r;
1648
1649         r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name);
1650         if (r)
1651                 return r;
1652
1653         return r;
1654 }
1655
1656 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1657 {
1658         u32 count = 0;
1659         const struct cs_section_def *sect = NULL;
1660         const struct cs_extent_def *ext = NULL;
1661
1662         /* begin clear state */
1663         count += 2;
1664         /* context control state */
1665         count += 3;
1666
1667         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1668                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1669                         if (sect->id == SECT_CONTEXT)
1670                                 count += 2 + ext->reg_count;
1671                         else
1672                                 return 0;
1673                 }
1674         }
1675
1676         /* end clear state */
1677         count += 2;
1678         /* clear state */
1679         count += 2;
1680
1681         return count;
1682 }
1683
1684 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1685                                     volatile u32 *buffer)
1686 {
1687         u32 count = 0, i;
1688         const struct cs_section_def *sect = NULL;
1689         const struct cs_extent_def *ext = NULL;
1690
1691         if (adev->gfx.rlc.cs_data == NULL)
1692                 return;
1693         if (buffer == NULL)
1694                 return;
1695
1696         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1697         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1698
1699         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1700         buffer[count++] = cpu_to_le32(0x80000000);
1701         buffer[count++] = cpu_to_le32(0x80000000);
1702
1703         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1704                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1705                         if (sect->id == SECT_CONTEXT) {
1706                                 buffer[count++] =
1707                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1708                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1709                                                 PACKET3_SET_CONTEXT_REG_START);
1710                                 for (i = 0; i < ext->reg_count; i++)
1711                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1712                         } else {
1713                                 return;
1714                         }
1715                 }
1716         }
1717
1718         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1719         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1720
1721         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1722         buffer[count++] = cpu_to_le32(0);
1723 }
1724
1725 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1726 {
1727         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1728         uint32_t pg_always_on_cu_num = 2;
1729         uint32_t always_on_cu_num;
1730         uint32_t i, j, k;
1731         uint32_t mask, cu_bitmap, counter;
1732
1733         if (adev->flags & AMD_IS_APU)
1734                 always_on_cu_num = 4;
1735         else if (adev->asic_type == CHIP_VEGA12)
1736                 always_on_cu_num = 8;
1737         else
1738                 always_on_cu_num = 12;
1739
1740         mutex_lock(&adev->grbm_idx_mutex);
1741         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1742                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1743                         mask = 1;
1744                         cu_bitmap = 0;
1745                         counter = 0;
1746                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1747
1748                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1749                                 if (cu_info->bitmap[i][j] & mask) {
1750                                         if (counter == pg_always_on_cu_num)
1751                                                 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1752                                         if (counter < always_on_cu_num)
1753                                                 cu_bitmap |= mask;
1754                                         else
1755                                                 break;
1756                                         counter++;
1757                                 }
1758                                 mask <<= 1;
1759                         }
1760
1761                         WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1762                         cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1763                 }
1764         }
1765         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1766         mutex_unlock(&adev->grbm_idx_mutex);
1767 }
1768
1769 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1770 {
1771         uint32_t data;
1772
1773         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1774         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1775         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1776         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1777         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1778
1779         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1780         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1781
1782         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1783         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1784
1785         mutex_lock(&adev->grbm_idx_mutex);
1786         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1787         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1788         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1789
1790         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1791         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1792         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1793         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1794         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1795
1796         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1797         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1798         data &= 0x0000FFFF;
1799         data |= 0x00C00000;
1800         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1801
1802         /*
1803          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1804          * programmed in gfx_v9_0_init_always_on_cu_mask()
1805          */
1806
1807         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1808          * but used for RLC_LB_CNTL configuration */
1809         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1810         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1811         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1812         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1813         mutex_unlock(&adev->grbm_idx_mutex);
1814
1815         gfx_v9_0_init_always_on_cu_mask(adev);
1816 }
1817
1818 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1819 {
1820         uint32_t data;
1821
1822         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1823         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1824         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1825         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1826         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1827
1828         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1829         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1830
1831         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1832         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1833
1834         mutex_lock(&adev->grbm_idx_mutex);
1835         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1836         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1837         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1838
1839         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1840         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1841         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1842         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1843         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1844
1845         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1846         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1847         data &= 0x0000FFFF;
1848         data |= 0x00C00000;
1849         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1850
1851         /*
1852          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1853          * programmed in gfx_v9_0_init_always_on_cu_mask()
1854          */
1855
1856         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1857          * but used for RLC_LB_CNTL configuration */
1858         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1859         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1860         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1861         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1862         mutex_unlock(&adev->grbm_idx_mutex);
1863
1864         gfx_v9_0_init_always_on_cu_mask(adev);
1865 }
1866
1867 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1868 {
1869         WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1870 }
1871
1872 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1873 {
1874         return 5;
1875 }
1876
1877 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1878 {
1879         const struct cs_section_def *cs_data;
1880         int r;
1881
1882         adev->gfx.rlc.cs_data = gfx9_cs_data;
1883
1884         cs_data = adev->gfx.rlc.cs_data;
1885
1886         if (cs_data) {
1887                 /* init clear state block */
1888                 r = amdgpu_gfx_rlc_init_csb(adev);
1889                 if (r)
1890                         return r;
1891         }
1892
1893         if (adev->flags & AMD_IS_APU) {
1894                 /* TODO: double check the cp_table_size for RV */
1895                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1896                 r = amdgpu_gfx_rlc_init_cpt(adev);
1897                 if (r)
1898                         return r;
1899         }
1900
1901         switch (adev->asic_type) {
1902         case CHIP_RAVEN:
1903                 gfx_v9_0_init_lbpw(adev);
1904                 break;
1905         case CHIP_VEGA20:
1906                 gfx_v9_4_init_lbpw(adev);
1907                 break;
1908         default:
1909                 break;
1910         }
1911
1912         /* init spm vmid with 0xf */
1913         if (adev->gfx.rlc.funcs->update_spm_vmid)
1914                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1915
1916         return 0;
1917 }
1918
1919 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1920 {
1921         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1922         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1923 }
1924
1925 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1926 {
1927         int r;
1928         u32 *hpd;
1929         const __le32 *fw_data;
1930         unsigned fw_size;
1931         u32 *fw;
1932         size_t mec_hpd_size;
1933
1934         const struct gfx_firmware_header_v1_0 *mec_hdr;
1935
1936         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1937
1938         /* take ownership of the relevant compute queues */
1939         amdgpu_gfx_compute_queue_acquire(adev);
1940         mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1941
1942         r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1943                                       AMDGPU_GEM_DOMAIN_VRAM,
1944                                       &adev->gfx.mec.hpd_eop_obj,
1945                                       &adev->gfx.mec.hpd_eop_gpu_addr,
1946                                       (void **)&hpd);
1947         if (r) {
1948                 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1949                 gfx_v9_0_mec_fini(adev);
1950                 return r;
1951         }
1952
1953         memset(hpd, 0, mec_hpd_size);
1954
1955         amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1956         amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1957
1958         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1959
1960         fw_data = (const __le32 *)
1961                 (adev->gfx.mec_fw->data +
1962                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1963         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1964
1965         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1966                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1967                                       &adev->gfx.mec.mec_fw_obj,
1968                                       &adev->gfx.mec.mec_fw_gpu_addr,
1969                                       (void **)&fw);
1970         if (r) {
1971                 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1972                 gfx_v9_0_mec_fini(adev);
1973                 return r;
1974         }
1975
1976         memcpy(fw, fw_data, fw_size);
1977
1978         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1979         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1980
1981         return 0;
1982 }
1983
1984 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1985 {
1986         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1987                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1988                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1989                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
1990                 (SQ_IND_INDEX__FORCE_READ_MASK));
1991         return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1992 }
1993
1994 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1995                            uint32_t wave, uint32_t thread,
1996                            uint32_t regno, uint32_t num, uint32_t *out)
1997 {
1998         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1999                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2000                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2001                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
2002                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
2003                 (SQ_IND_INDEX__FORCE_READ_MASK) |
2004                 (SQ_IND_INDEX__AUTO_INCR_MASK));
2005         while (num--)
2006                 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2007 }
2008
2009 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
2010 {
2011         /* type 1 wave data */
2012         dst[(*no_fields)++] = 1;
2013         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
2014         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
2015         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
2016         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
2017         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
2018         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
2019         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
2020         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
2021         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
2022         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
2023         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
2024         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
2025         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
2026         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
2027 }
2028
2029 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
2030                                      uint32_t wave, uint32_t start,
2031                                      uint32_t size, uint32_t *dst)
2032 {
2033         wave_read_regs(
2034                 adev, simd, wave, 0,
2035                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
2036 }
2037
2038 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
2039                                      uint32_t wave, uint32_t thread,
2040                                      uint32_t start, uint32_t size,
2041                                      uint32_t *dst)
2042 {
2043         wave_read_regs(
2044                 adev, simd, wave, thread,
2045                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
2046 }
2047
2048 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
2049                                   u32 me, u32 pipe, u32 q, u32 vm)
2050 {
2051         soc15_grbm_select(adev, me, pipe, q, vm);
2052 }
2053
2054 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2055         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2056         .select_se_sh = &gfx_v9_0_select_se_sh,
2057         .read_wave_data = &gfx_v9_0_read_wave_data,
2058         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2059         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2060         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2061         .ras_error_inject = &gfx_v9_0_ras_error_inject,
2062         .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2063         .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2064 };
2065
2066 static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = {
2067         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2068         .select_se_sh = &gfx_v9_0_select_se_sh,
2069         .read_wave_data = &gfx_v9_0_read_wave_data,
2070         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2071         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2072         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2073         .ras_error_inject = &gfx_v9_4_ras_error_inject,
2074         .query_ras_error_count = &gfx_v9_4_query_ras_error_count,
2075         .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count,
2076 };
2077
2078 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2079 {
2080         u32 gb_addr_config;
2081         int err;
2082
2083         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
2084
2085         switch (adev->asic_type) {
2086         case CHIP_VEGA10:
2087                 adev->gfx.config.max_hw_contexts = 8;
2088                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2089                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2090                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2091                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2092                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2093                 break;
2094         case CHIP_VEGA12:
2095                 adev->gfx.config.max_hw_contexts = 8;
2096                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2097                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2098                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2099                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2100                 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2101                 DRM_INFO("fix gfx.config for vega12\n");
2102                 break;
2103         case CHIP_VEGA20:
2104                 adev->gfx.config.max_hw_contexts = 8;
2105                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2106                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2107                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2108                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2109                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2110                 gb_addr_config &= ~0xf3e777ff;
2111                 gb_addr_config |= 0x22014042;
2112                 /* check vbios table if gpu info is not available */
2113                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2114                 if (err)
2115                         return err;
2116                 break;
2117         case CHIP_RAVEN:
2118                 adev->gfx.config.max_hw_contexts = 8;
2119                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2120                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2121                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2122                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2123                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2124                         gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2125                 else
2126                         gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2127                 break;
2128         case CHIP_ARCTURUS:
2129                 adev->gfx.funcs = &gfx_v9_4_gfx_funcs;
2130                 adev->gfx.config.max_hw_contexts = 8;
2131                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2132                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2133                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2134                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2135                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2136                 gb_addr_config &= ~0xf3e777ff;
2137                 gb_addr_config |= 0x22014042;
2138                 break;
2139         case CHIP_RENOIR:
2140                 adev->gfx.config.max_hw_contexts = 8;
2141                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2142                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2143                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2144                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2145                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2146                 gb_addr_config &= ~0xf3e777ff;
2147                 gb_addr_config |= 0x22010042;
2148                 break;
2149         default:
2150                 BUG();
2151                 break;
2152         }
2153
2154         adev->gfx.config.gb_addr_config = gb_addr_config;
2155
2156         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2157                         REG_GET_FIELD(
2158                                         adev->gfx.config.gb_addr_config,
2159                                         GB_ADDR_CONFIG,
2160                                         NUM_PIPES);
2161
2162         adev->gfx.config.max_tile_pipes =
2163                 adev->gfx.config.gb_addr_config_fields.num_pipes;
2164
2165         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2166                         REG_GET_FIELD(
2167                                         adev->gfx.config.gb_addr_config,
2168                                         GB_ADDR_CONFIG,
2169                                         NUM_BANKS);
2170         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2171                         REG_GET_FIELD(
2172                                         adev->gfx.config.gb_addr_config,
2173                                         GB_ADDR_CONFIG,
2174                                         MAX_COMPRESSED_FRAGS);
2175         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2176                         REG_GET_FIELD(
2177                                         adev->gfx.config.gb_addr_config,
2178                                         GB_ADDR_CONFIG,
2179                                         NUM_RB_PER_SE);
2180         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2181                         REG_GET_FIELD(
2182                                         adev->gfx.config.gb_addr_config,
2183                                         GB_ADDR_CONFIG,
2184                                         NUM_SHADER_ENGINES);
2185         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2186                         REG_GET_FIELD(
2187                                         adev->gfx.config.gb_addr_config,
2188                                         GB_ADDR_CONFIG,
2189                                         PIPE_INTERLEAVE_SIZE));
2190
2191         return 0;
2192 }
2193
2194 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2195                                       int mec, int pipe, int queue)
2196 {
2197         int r;
2198         unsigned irq_type;
2199         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2200         unsigned int hw_prio;
2201
2202         ring = &adev->gfx.compute_ring[ring_id];
2203
2204         /* mec0 is me1 */
2205         ring->me = mec + 1;
2206         ring->pipe = pipe;
2207         ring->queue = queue;
2208
2209         ring->ring_obj = NULL;
2210         ring->use_doorbell = true;
2211         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2212         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2213                                 + (ring_id * GFX9_MEC_HPD_SIZE);
2214         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2215
2216         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2217                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2218                 + ring->pipe;
2219         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->queue) ?
2220                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
2221         /* type-2 packets are deprecated on MEC, use type-3 instead */
2222         r = amdgpu_ring_init(adev, ring, 1024,
2223                              &adev->gfx.eop_irq, irq_type, hw_prio);
2224         if (r)
2225                 return r;
2226
2227
2228         return 0;
2229 }
2230
2231 static int gfx_v9_0_sw_init(void *handle)
2232 {
2233         int i, j, k, r, ring_id;
2234         struct amdgpu_ring *ring;
2235         struct amdgpu_kiq *kiq;
2236         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2237
2238         switch (adev->asic_type) {
2239         case CHIP_VEGA10:
2240         case CHIP_VEGA12:
2241         case CHIP_VEGA20:
2242         case CHIP_RAVEN:
2243         case CHIP_ARCTURUS:
2244         case CHIP_RENOIR:
2245                 adev->gfx.mec.num_mec = 2;
2246                 break;
2247         default:
2248                 adev->gfx.mec.num_mec = 1;
2249                 break;
2250         }
2251
2252         adev->gfx.mec.num_pipe_per_mec = 4;
2253         adev->gfx.mec.num_queue_per_pipe = 8;
2254
2255         /* EOP Event */
2256         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2257         if (r)
2258                 return r;
2259
2260         /* Privileged reg */
2261         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2262                               &adev->gfx.priv_reg_irq);
2263         if (r)
2264                 return r;
2265
2266         /* Privileged inst */
2267         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2268                               &adev->gfx.priv_inst_irq);
2269         if (r)
2270                 return r;
2271
2272         /* ECC error */
2273         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2274                               &adev->gfx.cp_ecc_error_irq);
2275         if (r)
2276                 return r;
2277
2278         /* FUE error */
2279         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2280                               &adev->gfx.cp_ecc_error_irq);
2281         if (r)
2282                 return r;
2283
2284         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2285
2286         gfx_v9_0_scratch_init(adev);
2287
2288         r = gfx_v9_0_init_microcode(adev);
2289         if (r) {
2290                 DRM_ERROR("Failed to load gfx firmware!\n");
2291                 return r;
2292         }
2293
2294         r = adev->gfx.rlc.funcs->init(adev);
2295         if (r) {
2296                 DRM_ERROR("Failed to init rlc BOs!\n");
2297                 return r;
2298         }
2299
2300         r = gfx_v9_0_mec_init(adev);
2301         if (r) {
2302                 DRM_ERROR("Failed to init MEC BOs!\n");
2303                 return r;
2304         }
2305
2306         /* set up the gfx ring */
2307         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2308                 ring = &adev->gfx.gfx_ring[i];
2309                 ring->ring_obj = NULL;
2310                 if (!i)
2311                         sprintf(ring->name, "gfx");
2312                 else
2313                         sprintf(ring->name, "gfx_%d", i);
2314                 ring->use_doorbell = true;
2315                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2316                 r = amdgpu_ring_init(adev, ring, 1024,
2317                                      &adev->gfx.eop_irq,
2318                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2319                                      AMDGPU_RING_PRIO_DEFAULT);
2320                 if (r)
2321                         return r;
2322         }
2323
2324         /* set up the compute queues - allocate horizontally across pipes */
2325         ring_id = 0;
2326         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2327                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2328                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2329                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2330                                         continue;
2331
2332                                 r = gfx_v9_0_compute_ring_init(adev,
2333                                                                ring_id,
2334                                                                i, k, j);
2335                                 if (r)
2336                                         return r;
2337
2338                                 ring_id++;
2339                         }
2340                 }
2341         }
2342
2343         r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2344         if (r) {
2345                 DRM_ERROR("Failed to init KIQ BOs!\n");
2346                 return r;
2347         }
2348
2349         kiq = &adev->gfx.kiq;
2350         r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2351         if (r)
2352                 return r;
2353
2354         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2355         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2356         if (r)
2357                 return r;
2358
2359         adev->gfx.ce_ram_size = 0x8000;
2360
2361         r = gfx_v9_0_gpu_early_init(adev);
2362         if (r)
2363                 return r;
2364
2365         return 0;
2366 }
2367
2368
2369 static int gfx_v9_0_sw_fini(void *handle)
2370 {
2371         int i;
2372         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2373
2374         amdgpu_gfx_ras_fini(adev);
2375
2376         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2377                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2378         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2379                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2380
2381         amdgpu_gfx_mqd_sw_fini(adev);
2382         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2383         amdgpu_gfx_kiq_fini(adev);
2384
2385         gfx_v9_0_mec_fini(adev);
2386         amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
2387         if (adev->flags & AMD_IS_APU) {
2388                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2389                                 &adev->gfx.rlc.cp_table_gpu_addr,
2390                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2391         }
2392         gfx_v9_0_free_microcode(adev);
2393
2394         return 0;
2395 }
2396
2397
2398 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2399 {
2400         /* TODO */
2401 }
2402
2403 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance)
2404 {
2405         u32 data;
2406
2407         if (instance == 0xffffffff)
2408                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2409         else
2410                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2411
2412         if (se_num == 0xffffffff)
2413                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2414         else
2415                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2416
2417         if (sh_num == 0xffffffff)
2418                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2419         else
2420                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2421
2422         WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2423 }
2424
2425 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2426 {
2427         u32 data, mask;
2428
2429         data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2430         data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2431
2432         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2433         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2434
2435         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2436                                          adev->gfx.config.max_sh_per_se);
2437
2438         return (~data) & mask;
2439 }
2440
2441 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2442 {
2443         int i, j;
2444         u32 data;
2445         u32 active_rbs = 0;
2446         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2447                                         adev->gfx.config.max_sh_per_se;
2448
2449         mutex_lock(&adev->grbm_idx_mutex);
2450         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2451                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2452                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2453                         data = gfx_v9_0_get_rb_active_bitmap(adev);
2454                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2455                                                rb_bitmap_width_per_sh);
2456                 }
2457         }
2458         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2459         mutex_unlock(&adev->grbm_idx_mutex);
2460
2461         adev->gfx.config.backend_enable_mask = active_rbs;
2462         adev->gfx.config.num_rbs = hweight32(active_rbs);
2463 }
2464
2465 #define DEFAULT_SH_MEM_BASES    (0x6000)
2466 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2467 {
2468         int i;
2469         uint32_t sh_mem_config;
2470         uint32_t sh_mem_bases;
2471
2472         /*
2473          * Configure apertures:
2474          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
2475          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
2476          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
2477          */
2478         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2479
2480         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2481                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2482                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2483
2484         mutex_lock(&adev->srbm_mutex);
2485         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2486                 soc15_grbm_select(adev, 0, 0, 0, i);
2487                 /* CP and shaders */
2488                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2489                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2490         }
2491         soc15_grbm_select(adev, 0, 0, 0, 0);
2492         mutex_unlock(&adev->srbm_mutex);
2493
2494         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2495            acccess. These should be enabled by FW for target VMIDs. */
2496         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2497                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2498                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2499                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2500                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2501         }
2502 }
2503
2504 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2505 {
2506         int vmid;
2507
2508         /*
2509          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2510          * access. Compute VMIDs should be enabled by FW for target VMIDs,
2511          * the driver can enable them for graphics. VMID0 should maintain
2512          * access so that HWS firmware can save/restore entries.
2513          */
2514         for (vmid = 1; vmid < 16; vmid++) {
2515                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2516                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2517                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2518                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2519         }
2520 }
2521
2522 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2523 {
2524         uint32_t tmp;
2525
2526         switch (adev->asic_type) {
2527         case CHIP_ARCTURUS:
2528                 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2529                 tmp = REG_SET_FIELD(tmp, SQ_CONFIG,
2530                                         DISABLE_BARRIER_WAITCNT, 1);
2531                 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2532                 break;
2533         default:
2534                 break;
2535         }
2536 }
2537
2538 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2539 {
2540         u32 tmp;
2541         int i;
2542
2543         WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2544
2545         gfx_v9_0_tiling_mode_table_init(adev);
2546
2547         gfx_v9_0_setup_rb(adev);
2548         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2549         adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2550
2551         /* XXX SH_MEM regs */
2552         /* where to put LDS, scratch, GPUVM in FSA64 space */
2553         mutex_lock(&adev->srbm_mutex);
2554         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2555                 soc15_grbm_select(adev, 0, 0, 0, i);
2556                 /* CP and shaders */
2557                 if (i == 0) {
2558                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2559                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2560                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2561                                             !!amdgpu_noretry);
2562                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2563                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2564                 } else {
2565                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2566                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2567                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2568                                             !!amdgpu_noretry);
2569                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2570                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2571                                 (adev->gmc.private_aperture_start >> 48));
2572                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2573                                 (adev->gmc.shared_aperture_start >> 48));
2574                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2575                 }
2576         }
2577         soc15_grbm_select(adev, 0, 0, 0, 0);
2578
2579         mutex_unlock(&adev->srbm_mutex);
2580
2581         gfx_v9_0_init_compute_vmid(adev);
2582         gfx_v9_0_init_gds_vmid(adev);
2583         gfx_v9_0_init_sq_config(adev);
2584 }
2585
2586 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2587 {
2588         u32 i, j, k;
2589         u32 mask;
2590
2591         mutex_lock(&adev->grbm_idx_mutex);
2592         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2593                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2594                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2595                         for (k = 0; k < adev->usec_timeout; k++) {
2596                                 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2597                                         break;
2598                                 udelay(1);
2599                         }
2600                         if (k == adev->usec_timeout) {
2601                                 gfx_v9_0_select_se_sh(adev, 0xffffffff,
2602                                                       0xffffffff, 0xffffffff);
2603                                 mutex_unlock(&adev->grbm_idx_mutex);
2604                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2605                                          i, j);
2606                                 return;
2607                         }
2608                 }
2609         }
2610         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2611         mutex_unlock(&adev->grbm_idx_mutex);
2612
2613         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2614                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2615                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2616                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2617         for (k = 0; k < adev->usec_timeout; k++) {
2618                 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2619                         break;
2620                 udelay(1);
2621         }
2622 }
2623
2624 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2625                                                bool enable)
2626 {
2627         u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2628
2629         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2630         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2631         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2632         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2633
2634         WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2635 }
2636
2637 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2638 {
2639         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2640         /* csib */
2641         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2642                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
2643         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2644                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2645         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2646                         adev->gfx.rlc.clear_state_size);
2647 }
2648
2649 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2650                                 int indirect_offset,
2651                                 int list_size,
2652                                 int *unique_indirect_regs,
2653                                 int unique_indirect_reg_count,
2654                                 int *indirect_start_offsets,
2655                                 int *indirect_start_offsets_count,
2656                                 int max_start_offsets_count)
2657 {
2658         int idx;
2659
2660         for (; indirect_offset < list_size; indirect_offset++) {
2661                 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2662                 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2663                 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2664
2665                 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2666                         indirect_offset += 2;
2667
2668                         /* look for the matching indice */
2669                         for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2670                                 if (unique_indirect_regs[idx] ==
2671                                         register_list_format[indirect_offset] ||
2672                                         !unique_indirect_regs[idx])
2673                                         break;
2674                         }
2675
2676                         BUG_ON(idx >= unique_indirect_reg_count);
2677
2678                         if (!unique_indirect_regs[idx])
2679                                 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2680
2681                         indirect_offset++;
2682                 }
2683         }
2684 }
2685
2686 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2687 {
2688         int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2689         int unique_indirect_reg_count = 0;
2690
2691         int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2692         int indirect_start_offsets_count = 0;
2693
2694         int list_size = 0;
2695         int i = 0, j = 0;
2696         u32 tmp = 0;
2697
2698         u32 *register_list_format =
2699                 kmemdup(adev->gfx.rlc.register_list_format,
2700                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2701         if (!register_list_format)
2702                 return -ENOMEM;
2703
2704         /* setup unique_indirect_regs array and indirect_start_offsets array */
2705         unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2706         gfx_v9_1_parse_ind_reg_list(register_list_format,
2707                                     adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2708                                     adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2709                                     unique_indirect_regs,
2710                                     unique_indirect_reg_count,
2711                                     indirect_start_offsets,
2712                                     &indirect_start_offsets_count,
2713                                     ARRAY_SIZE(indirect_start_offsets));
2714
2715         /* enable auto inc in case it is disabled */
2716         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2717         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2718         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2719
2720         /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2721         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2722                 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2723         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2724                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2725                         adev->gfx.rlc.register_restore[i]);
2726
2727         /* load indirect register */
2728         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2729                 adev->gfx.rlc.reg_list_format_start);
2730
2731         /* direct register portion */
2732         for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2733                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2734                         register_list_format[i]);
2735
2736         /* indirect register portion */
2737         while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2738                 if (register_list_format[i] == 0xFFFFFFFF) {
2739                         WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2740                         continue;
2741                 }
2742
2743                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2744                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2745
2746                 for (j = 0; j < unique_indirect_reg_count; j++) {
2747                         if (register_list_format[i] == unique_indirect_regs[j]) {
2748                                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2749                                 break;
2750                         }
2751                 }
2752
2753                 BUG_ON(j >= unique_indirect_reg_count);
2754
2755                 i++;
2756         }
2757
2758         /* set save/restore list size */
2759         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2760         list_size = list_size >> 1;
2761         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2762                 adev->gfx.rlc.reg_restore_list_size);
2763         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2764
2765         /* write the starting offsets to RLC scratch ram */
2766         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2767                 adev->gfx.rlc.starting_offsets_start);
2768         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2769                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2770                        indirect_start_offsets[i]);
2771
2772         /* load unique indirect regs*/
2773         for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2774                 if (unique_indirect_regs[i] != 0) {
2775                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2776                                + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2777                                unique_indirect_regs[i] & 0x3FFFF);
2778
2779                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2780                                + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2781                                unique_indirect_regs[i] >> 20);
2782                 }
2783         }
2784
2785         kfree(register_list_format);
2786         return 0;
2787 }
2788
2789 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2790 {
2791         WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2792 }
2793
2794 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2795                                              bool enable)
2796 {
2797         uint32_t data = 0;
2798         uint32_t default_data = 0;
2799
2800         default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2801         if (enable == true) {
2802                 /* enable GFXIP control over CGPG */
2803                 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2804                 if(default_data != data)
2805                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2806
2807                 /* update status */
2808                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2809                 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2810                 if(default_data != data)
2811                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2812         } else {
2813                 /* restore GFXIP control over GCPG */
2814                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2815                 if(default_data != data)
2816                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2817         }
2818 }
2819
2820 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2821 {
2822         uint32_t data = 0;
2823
2824         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2825                               AMD_PG_SUPPORT_GFX_SMG |
2826                               AMD_PG_SUPPORT_GFX_DMG)) {
2827                 /* init IDLE_POLL_COUNT = 60 */
2828                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2829                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2830                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2831                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2832
2833                 /* init RLC PG Delay */
2834                 data = 0;
2835                 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2836                 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2837                 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2838                 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2839                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2840
2841                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2842                 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2843                 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2844                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2845
2846                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2847                 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2848                 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2849                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2850
2851                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2852                 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2853
2854                 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2855                 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2856                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2857                 if (adev->asic_type != CHIP_RENOIR)
2858                         pwr_10_0_gfxip_control_over_cgpg(adev, true);
2859         }
2860 }
2861
2862 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2863                                                 bool enable)
2864 {
2865         uint32_t data = 0;
2866         uint32_t default_data = 0;
2867
2868         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2869         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2870                              SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2871                              enable ? 1 : 0);
2872         if (default_data != data)
2873                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2874 }
2875
2876 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2877                                                 bool enable)
2878 {
2879         uint32_t data = 0;
2880         uint32_t default_data = 0;
2881
2882         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2883         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2884                              SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2885                              enable ? 1 : 0);
2886         if(default_data != data)
2887                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2888 }
2889
2890 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2891                                         bool enable)
2892 {
2893         uint32_t data = 0;
2894         uint32_t default_data = 0;
2895
2896         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2897         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2898                              CP_PG_DISABLE,
2899                              enable ? 0 : 1);
2900         if(default_data != data)
2901                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2902 }
2903
2904 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2905                                                 bool enable)
2906 {
2907         uint32_t data, default_data;
2908
2909         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2910         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2911                              GFX_POWER_GATING_ENABLE,
2912                              enable ? 1 : 0);
2913         if(default_data != data)
2914                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2915 }
2916
2917 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2918                                                 bool enable)
2919 {
2920         uint32_t data, default_data;
2921
2922         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2923         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2924                              GFX_PIPELINE_PG_ENABLE,
2925                              enable ? 1 : 0);
2926         if(default_data != data)
2927                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2928
2929         if (!enable)
2930                 /* read any GFX register to wake up GFX */
2931                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2932 }
2933
2934 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
2935                                                        bool enable)
2936 {
2937         uint32_t data, default_data;
2938
2939         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2940         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2941                              STATIC_PER_CU_PG_ENABLE,
2942                              enable ? 1 : 0);
2943         if(default_data != data)
2944                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2945 }
2946
2947 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
2948                                                 bool enable)
2949 {
2950         uint32_t data, default_data;
2951
2952         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2953         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2954                              DYN_PER_CU_PG_ENABLE,
2955                              enable ? 1 : 0);
2956         if(default_data != data)
2957                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2958 }
2959
2960 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
2961 {
2962         gfx_v9_0_init_csb(adev);
2963
2964         /*
2965          * Rlc save restore list is workable since v2_1.
2966          * And it's needed by gfxoff feature.
2967          */
2968         if (adev->gfx.rlc.is_rlc_v2_1) {
2969                 if (adev->asic_type == CHIP_VEGA12 ||
2970                     (adev->apu_flags & AMD_APU_IS_RAVEN2))
2971                         gfx_v9_1_init_rlc_save_restore_list(adev);
2972                 gfx_v9_0_enable_save_restore_machine(adev);
2973         }
2974
2975         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2976                               AMD_PG_SUPPORT_GFX_SMG |
2977                               AMD_PG_SUPPORT_GFX_DMG |
2978                               AMD_PG_SUPPORT_CP |
2979                               AMD_PG_SUPPORT_GDS |
2980                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
2981                 WREG32(mmRLC_JUMP_TABLE_RESTORE,
2982                        adev->gfx.rlc.cp_table_gpu_addr >> 8);
2983                 gfx_v9_0_init_gfx_power_gating(adev);
2984         }
2985 }
2986
2987 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
2988 {
2989         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
2990         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
2991         gfx_v9_0_wait_for_rlc_serdes(adev);
2992 }
2993
2994 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
2995 {
2996         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2997         udelay(50);
2998         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2999         udelay(50);
3000 }
3001
3002 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3003 {
3004 #ifdef AMDGPU_RLC_DEBUG_RETRY
3005         u32 rlc_ucode_ver;
3006 #endif
3007
3008         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3009         udelay(50);
3010
3011         /* carrizo do enable cp interrupt after cp inited */
3012         if (!(adev->flags & AMD_IS_APU)) {
3013                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3014                 udelay(50);
3015         }
3016
3017 #ifdef AMDGPU_RLC_DEBUG_RETRY
3018         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3019         rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3020         if(rlc_ucode_ver == 0x108) {
3021                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3022                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3023                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3024                  * default is 0x9C4 to create a 100us interval */
3025                 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3026                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3027                  * to disable the page fault retry interrupts, default is
3028                  * 0x100 (256) */
3029                 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3030         }
3031 #endif
3032 }
3033
3034 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3035 {
3036         const struct rlc_firmware_header_v2_0 *hdr;
3037         const __le32 *fw_data;
3038         unsigned i, fw_size;
3039
3040         if (!adev->gfx.rlc_fw)
3041                 return -EINVAL;
3042
3043         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3044         amdgpu_ucode_print_rlc_hdr(&hdr->header);
3045
3046         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3047                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3048         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3049
3050         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3051                         RLCG_UCODE_LOADING_START_ADDRESS);
3052         for (i = 0; i < fw_size; i++)
3053                 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3054         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3055
3056         return 0;
3057 }
3058
3059 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3060 {
3061         int r;
3062
3063         if (amdgpu_sriov_vf(adev)) {
3064                 gfx_v9_0_init_csb(adev);
3065                 return 0;
3066         }
3067
3068         adev->gfx.rlc.funcs->stop(adev);
3069
3070         /* disable CG */
3071         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3072
3073         gfx_v9_0_init_pg(adev);
3074
3075         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3076                 /* legacy rlc firmware loading */
3077                 r = gfx_v9_0_rlc_load_microcode(adev);
3078                 if (r)
3079                         return r;
3080         }
3081
3082         switch (adev->asic_type) {
3083         case CHIP_RAVEN:
3084                 if (amdgpu_lbpw == 0)
3085                         gfx_v9_0_enable_lbpw(adev, false);
3086                 else
3087                         gfx_v9_0_enable_lbpw(adev, true);
3088                 break;
3089         case CHIP_VEGA20:
3090                 if (amdgpu_lbpw > 0)
3091                         gfx_v9_0_enable_lbpw(adev, true);
3092                 else
3093                         gfx_v9_0_enable_lbpw(adev, false);
3094                 break;
3095         default:
3096                 break;
3097         }
3098
3099         adev->gfx.rlc.funcs->start(adev);
3100
3101         return 0;
3102 }
3103
3104 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3105 {
3106         u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3107
3108         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3109         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3110         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3111         WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3112         udelay(50);
3113 }
3114
3115 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3116 {
3117         const struct gfx_firmware_header_v1_0 *pfp_hdr;
3118         const struct gfx_firmware_header_v1_0 *ce_hdr;
3119         const struct gfx_firmware_header_v1_0 *me_hdr;
3120         const __le32 *fw_data;
3121         unsigned i, fw_size;
3122
3123         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3124                 return -EINVAL;
3125
3126         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3127                 adev->gfx.pfp_fw->data;
3128         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3129                 adev->gfx.ce_fw->data;
3130         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3131                 adev->gfx.me_fw->data;
3132
3133         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3134         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3135         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3136
3137         gfx_v9_0_cp_gfx_enable(adev, false);
3138
3139         /* PFP */
3140         fw_data = (const __le32 *)
3141                 (adev->gfx.pfp_fw->data +
3142                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3143         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3144         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3145         for (i = 0; i < fw_size; i++)
3146                 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3147         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3148
3149         /* CE */
3150         fw_data = (const __le32 *)
3151                 (adev->gfx.ce_fw->data +
3152                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3153         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3154         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3155         for (i = 0; i < fw_size; i++)
3156                 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3157         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3158
3159         /* ME */
3160         fw_data = (const __le32 *)
3161                 (adev->gfx.me_fw->data +
3162                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3163         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3164         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3165         for (i = 0; i < fw_size; i++)
3166                 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3167         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3168
3169         return 0;
3170 }
3171
3172 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3173 {
3174         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3175         const struct cs_section_def *sect = NULL;
3176         const struct cs_extent_def *ext = NULL;
3177         int r, i, tmp;
3178
3179         /* init the CP */
3180         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3181         WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3182
3183         gfx_v9_0_cp_gfx_enable(adev, true);
3184
3185         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3186         if (r) {
3187                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3188                 return r;
3189         }
3190
3191         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3192         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3193
3194         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3195         amdgpu_ring_write(ring, 0x80000000);
3196         amdgpu_ring_write(ring, 0x80000000);
3197
3198         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3199                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3200                         if (sect->id == SECT_CONTEXT) {
3201                                 amdgpu_ring_write(ring,
3202                                        PACKET3(PACKET3_SET_CONTEXT_REG,
3203                                                ext->reg_count));
3204                                 amdgpu_ring_write(ring,
3205                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3206                                 for (i = 0; i < ext->reg_count; i++)
3207                                         amdgpu_ring_write(ring, ext->extent[i]);
3208                         }
3209                 }
3210         }
3211
3212         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3213         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3214
3215         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3216         amdgpu_ring_write(ring, 0);
3217
3218         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3219         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3220         amdgpu_ring_write(ring, 0x8000);
3221         amdgpu_ring_write(ring, 0x8000);
3222
3223         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3224         tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3225                 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3226         amdgpu_ring_write(ring, tmp);
3227         amdgpu_ring_write(ring, 0);
3228
3229         amdgpu_ring_commit(ring);
3230
3231         return 0;
3232 }
3233
3234 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3235 {
3236         struct amdgpu_ring *ring;
3237         u32 tmp;
3238         u32 rb_bufsz;
3239         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3240
3241         /* Set the write pointer delay */
3242         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3243
3244         /* set the RB to use vmid 0 */
3245         WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3246
3247         /* Set ring buffer size */
3248         ring = &adev->gfx.gfx_ring[0];
3249         rb_bufsz = order_base_2(ring->ring_size / 8);
3250         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3251         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3252 #ifdef __BIG_ENDIAN
3253         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3254 #endif
3255         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3256
3257         /* Initialize the ring buffer's write pointers */
3258         ring->wptr = 0;
3259         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3260         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3261
3262         /* set the wb address wether it's enabled or not */
3263         rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3264         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3265         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3266
3267         wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3268         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3269         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3270
3271         mdelay(1);
3272         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3273
3274         rb_addr = ring->gpu_addr >> 8;
3275         WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3276         WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3277
3278         tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3279         if (ring->use_doorbell) {
3280                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3281                                     DOORBELL_OFFSET, ring->doorbell_index);
3282                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3283                                     DOORBELL_EN, 1);
3284         } else {
3285                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3286         }
3287         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3288
3289         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3290                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
3291         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3292
3293         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3294                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3295
3296
3297         /* start the ring */
3298         gfx_v9_0_cp_gfx_start(adev);
3299         ring->sched.ready = true;
3300
3301         return 0;
3302 }
3303
3304 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3305 {
3306         if (enable) {
3307                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3308         } else {
3309                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3310                         (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3311                 adev->gfx.kiq.ring.sched.ready = false;
3312         }
3313         udelay(50);
3314 }
3315
3316 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3317 {
3318         const struct gfx_firmware_header_v1_0 *mec_hdr;
3319         const __le32 *fw_data;
3320         unsigned i;
3321         u32 tmp;
3322
3323         if (!adev->gfx.mec_fw)
3324                 return -EINVAL;
3325
3326         gfx_v9_0_cp_compute_enable(adev, false);
3327
3328         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3329         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3330
3331         fw_data = (const __le32 *)
3332                 (adev->gfx.mec_fw->data +
3333                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3334         tmp = 0;
3335         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3336         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3337         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3338
3339         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3340                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3341         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3342                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3343
3344         /* MEC1 */
3345         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3346                          mec_hdr->jt_offset);
3347         for (i = 0; i < mec_hdr->jt_size; i++)
3348                 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3349                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3350
3351         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3352                         adev->gfx.mec_fw_version);
3353         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3354
3355         return 0;
3356 }
3357
3358 /* KIQ functions */
3359 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3360 {
3361         uint32_t tmp;
3362         struct amdgpu_device *adev = ring->adev;
3363
3364         /* tell RLC which is KIQ queue */
3365         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3366         tmp &= 0xffffff00;
3367         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3368         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3369         tmp |= 0x80;
3370         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3371 }
3372
3373 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3374 {
3375         struct amdgpu_device *adev = ring->adev;
3376
3377         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3378                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring->queue)) {
3379                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3380                         mqd->cp_hqd_queue_priority =
3381                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3382                 }
3383         }
3384 }
3385
3386 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3387 {
3388         struct amdgpu_device *adev = ring->adev;
3389         struct v9_mqd *mqd = ring->mqd_ptr;
3390         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3391         uint32_t tmp;
3392
3393         mqd->header = 0xC0310800;
3394         mqd->compute_pipelinestat_enable = 0x00000001;
3395         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3396         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3397         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3398         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3399         mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3400         mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3401         mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3402         mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3403         mqd->compute_misc_reserved = 0x00000003;
3404
3405         mqd->dynamic_cu_mask_addr_lo =
3406                 lower_32_bits(ring->mqd_gpu_addr
3407                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3408         mqd->dynamic_cu_mask_addr_hi =
3409                 upper_32_bits(ring->mqd_gpu_addr
3410                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3411
3412         eop_base_addr = ring->eop_gpu_addr >> 8;
3413         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3414         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3415
3416         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3417         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3418         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3419                         (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3420
3421         mqd->cp_hqd_eop_control = tmp;
3422
3423         /* enable doorbell? */
3424         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3425
3426         if (ring->use_doorbell) {
3427                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3428                                     DOORBELL_OFFSET, ring->doorbell_index);
3429                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3430                                     DOORBELL_EN, 1);
3431                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3432                                     DOORBELL_SOURCE, 0);
3433                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3434                                     DOORBELL_HIT, 0);
3435         } else {
3436                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3437                                          DOORBELL_EN, 0);
3438         }
3439
3440         mqd->cp_hqd_pq_doorbell_control = tmp;
3441
3442         /* disable the queue if it's active */
3443         ring->wptr = 0;
3444         mqd->cp_hqd_dequeue_request = 0;
3445         mqd->cp_hqd_pq_rptr = 0;
3446         mqd->cp_hqd_pq_wptr_lo = 0;
3447         mqd->cp_hqd_pq_wptr_hi = 0;
3448
3449         /* set the pointer to the MQD */
3450         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3451         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3452
3453         /* set MQD vmid to 0 */
3454         tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3455         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3456         mqd->cp_mqd_control = tmp;
3457
3458         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3459         hqd_gpu_addr = ring->gpu_addr >> 8;
3460         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3461         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3462
3463         /* set up the HQD, this is similar to CP_RB0_CNTL */
3464         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3465         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3466                             (order_base_2(ring->ring_size / 4) - 1));
3467         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3468                         ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3469 #ifdef __BIG_ENDIAN
3470         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3471 #endif
3472         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3473         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3474         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3475         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3476         mqd->cp_hqd_pq_control = tmp;
3477
3478         /* set the wb address whether it's enabled or not */
3479         wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3480         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3481         mqd->cp_hqd_pq_rptr_report_addr_hi =
3482                 upper_32_bits(wb_gpu_addr) & 0xffff;
3483
3484         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3485         wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3486         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3487         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3488
3489         tmp = 0;
3490         /* enable the doorbell if requested */
3491         if (ring->use_doorbell) {
3492                 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3493                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3494                                 DOORBELL_OFFSET, ring->doorbell_index);
3495
3496                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3497                                          DOORBELL_EN, 1);
3498                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3499                                          DOORBELL_SOURCE, 0);
3500                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3501                                          DOORBELL_HIT, 0);
3502         }
3503
3504         mqd->cp_hqd_pq_doorbell_control = tmp;
3505
3506         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3507         ring->wptr = 0;
3508         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3509
3510         /* set the vmid for the queue */
3511         mqd->cp_hqd_vmid = 0;
3512
3513         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3514         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3515         mqd->cp_hqd_persistent_state = tmp;
3516
3517         /* set MIN_IB_AVAIL_SIZE */
3518         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3519         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3520         mqd->cp_hqd_ib_control = tmp;
3521
3522         /* set static priority for a queue/ring */
3523         gfx_v9_0_mqd_set_priority(ring, mqd);
3524         mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
3525
3526         /* map_queues packet doesn't need activate the queue,
3527          * so only kiq need set this field.
3528          */
3529         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3530                 mqd->cp_hqd_active = 1;
3531
3532         return 0;
3533 }
3534
3535 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3536 {
3537         struct amdgpu_device *adev = ring->adev;
3538         struct v9_mqd *mqd = ring->mqd_ptr;
3539         int j;
3540
3541         /* disable wptr polling */
3542         WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3543
3544         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3545                mqd->cp_hqd_eop_base_addr_lo);
3546         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3547                mqd->cp_hqd_eop_base_addr_hi);
3548
3549         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3550         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3551                mqd->cp_hqd_eop_control);
3552
3553         /* enable doorbell? */
3554         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3555                mqd->cp_hqd_pq_doorbell_control);
3556
3557         /* disable the queue if it's active */
3558         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3559                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3560                 for (j = 0; j < adev->usec_timeout; j++) {
3561                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3562                                 break;
3563                         udelay(1);
3564                 }
3565                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3566                        mqd->cp_hqd_dequeue_request);
3567                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3568                        mqd->cp_hqd_pq_rptr);
3569                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3570                        mqd->cp_hqd_pq_wptr_lo);
3571                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3572                        mqd->cp_hqd_pq_wptr_hi);
3573         }
3574
3575         /* set the pointer to the MQD */
3576         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3577                mqd->cp_mqd_base_addr_lo);
3578         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3579                mqd->cp_mqd_base_addr_hi);
3580
3581         /* set MQD vmid to 0 */
3582         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3583                mqd->cp_mqd_control);
3584
3585         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3586         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3587                mqd->cp_hqd_pq_base_lo);
3588         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3589                mqd->cp_hqd_pq_base_hi);
3590
3591         /* set up the HQD, this is similar to CP_RB0_CNTL */
3592         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3593                mqd->cp_hqd_pq_control);
3594
3595         /* set the wb address whether it's enabled or not */
3596         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3597                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3598         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3599                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3600
3601         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3602         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3603                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3604         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3605                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3606
3607         /* enable the doorbell if requested */
3608         if (ring->use_doorbell) {
3609                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3610                                         (adev->doorbell_index.kiq * 2) << 2);
3611                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3612                                         (adev->doorbell_index.userqueue_end * 2) << 2);
3613         }
3614
3615         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3616                mqd->cp_hqd_pq_doorbell_control);
3617
3618         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3619         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3620                mqd->cp_hqd_pq_wptr_lo);
3621         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3622                mqd->cp_hqd_pq_wptr_hi);
3623
3624         /* set the vmid for the queue */
3625         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3626
3627         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3628                mqd->cp_hqd_persistent_state);
3629
3630         /* activate the queue */
3631         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3632                mqd->cp_hqd_active);
3633
3634         if (ring->use_doorbell)
3635                 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3636
3637         return 0;
3638 }
3639
3640 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3641 {
3642         struct amdgpu_device *adev = ring->adev;
3643         int j;
3644
3645         /* disable the queue if it's active */
3646         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3647
3648                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3649
3650                 for (j = 0; j < adev->usec_timeout; j++) {
3651                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3652                                 break;
3653                         udelay(1);
3654                 }
3655
3656                 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3657                         DRM_DEBUG("KIQ dequeue request failed.\n");
3658
3659                         /* Manual disable if dequeue request times out */
3660                         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3661                 }
3662
3663                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3664                       0);
3665         }
3666
3667         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3668         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3669         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3670         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3671         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3672         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3673         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3674         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3675
3676         return 0;
3677 }
3678
3679 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3680 {
3681         struct amdgpu_device *adev = ring->adev;
3682         struct v9_mqd *mqd = ring->mqd_ptr;
3683         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3684
3685         gfx_v9_0_kiq_setting(ring);
3686
3687         if (adev->in_gpu_reset) { /* for GPU_RESET case */
3688                 /* reset MQD to a clean status */
3689                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3690                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3691
3692                 /* reset ring buffer */
3693                 ring->wptr = 0;
3694                 amdgpu_ring_clear_ring(ring);
3695
3696                 mutex_lock(&adev->srbm_mutex);
3697                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3698                 gfx_v9_0_kiq_init_register(ring);
3699                 soc15_grbm_select(adev, 0, 0, 0, 0);
3700                 mutex_unlock(&adev->srbm_mutex);
3701         } else {
3702                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3703                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3704                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3705                 mutex_lock(&adev->srbm_mutex);
3706                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3707                 gfx_v9_0_mqd_init(ring);
3708                 gfx_v9_0_kiq_init_register(ring);
3709                 soc15_grbm_select(adev, 0, 0, 0, 0);
3710                 mutex_unlock(&adev->srbm_mutex);
3711
3712                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3713                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3714         }
3715
3716         return 0;
3717 }
3718
3719 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3720 {
3721         struct amdgpu_device *adev = ring->adev;
3722         struct v9_mqd *mqd = ring->mqd_ptr;
3723         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3724
3725         if (!adev->in_gpu_reset && !adev->in_suspend) {
3726                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3727                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3728                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3729                 mutex_lock(&adev->srbm_mutex);
3730                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3731                 gfx_v9_0_mqd_init(ring);
3732                 soc15_grbm_select(adev, 0, 0, 0, 0);
3733                 mutex_unlock(&adev->srbm_mutex);
3734
3735                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3736                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3737         } else if (adev->in_gpu_reset) { /* for GPU_RESET case */
3738                 /* reset MQD to a clean status */
3739                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3740                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3741
3742                 /* reset ring buffer */
3743                 ring->wptr = 0;
3744                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
3745                 amdgpu_ring_clear_ring(ring);
3746         } else {
3747                 amdgpu_ring_clear_ring(ring);
3748         }
3749
3750         return 0;
3751 }
3752
3753 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3754 {
3755         struct amdgpu_ring *ring;
3756         int r;
3757
3758         ring = &adev->gfx.kiq.ring;
3759
3760         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3761         if (unlikely(r != 0))
3762                 return r;
3763
3764         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3765         if (unlikely(r != 0))
3766                 return r;
3767
3768         gfx_v9_0_kiq_init_queue(ring);
3769         amdgpu_bo_kunmap(ring->mqd_obj);
3770         ring->mqd_ptr = NULL;
3771         amdgpu_bo_unreserve(ring->mqd_obj);
3772         ring->sched.ready = true;
3773         return 0;
3774 }
3775
3776 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3777 {
3778         struct amdgpu_ring *ring = NULL;
3779         int r = 0, i;
3780
3781         gfx_v9_0_cp_compute_enable(adev, true);
3782
3783         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3784                 ring = &adev->gfx.compute_ring[i];
3785
3786                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3787                 if (unlikely(r != 0))
3788                         goto done;
3789                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3790                 if (!r) {
3791                         r = gfx_v9_0_kcq_init_queue(ring);
3792                         amdgpu_bo_kunmap(ring->mqd_obj);
3793                         ring->mqd_ptr = NULL;
3794                 }
3795                 amdgpu_bo_unreserve(ring->mqd_obj);
3796                 if (r)
3797                         goto done;
3798         }
3799
3800         r = amdgpu_gfx_enable_kcq(adev);
3801 done:
3802         return r;
3803 }
3804
3805 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3806 {
3807         int r, i;
3808         struct amdgpu_ring *ring;
3809
3810         if (!(adev->flags & AMD_IS_APU))
3811                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3812
3813         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3814                 if (adev->asic_type != CHIP_ARCTURUS) {
3815                         /* legacy firmware loading */
3816                         r = gfx_v9_0_cp_gfx_load_microcode(adev);
3817                         if (r)
3818                                 return r;
3819                 }
3820
3821                 r = gfx_v9_0_cp_compute_load_microcode(adev);
3822                 if (r)
3823                         return r;
3824         }
3825
3826         r = gfx_v9_0_kiq_resume(adev);
3827         if (r)
3828                 return r;
3829
3830         if (adev->asic_type != CHIP_ARCTURUS) {
3831                 r = gfx_v9_0_cp_gfx_resume(adev);
3832                 if (r)
3833                         return r;
3834         }
3835
3836         r = gfx_v9_0_kcq_resume(adev);
3837         if (r)
3838                 return r;
3839
3840         if (adev->asic_type != CHIP_ARCTURUS) {
3841                 ring = &adev->gfx.gfx_ring[0];
3842                 r = amdgpu_ring_test_helper(ring);
3843                 if (r)
3844                         return r;
3845         }
3846
3847         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3848                 ring = &adev->gfx.compute_ring[i];
3849                 amdgpu_ring_test_helper(ring);
3850         }
3851
3852         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3853
3854         return 0;
3855 }
3856
3857 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3858 {
3859         u32 tmp;
3860
3861         if (adev->asic_type != CHIP_ARCTURUS)
3862                 return;
3863
3864         tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3865         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3866                                 adev->df.hash_status.hash_64k);
3867         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3868                                 adev->df.hash_status.hash_2m);
3869         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3870                                 adev->df.hash_status.hash_1g);
3871         WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3872 }
3873
3874 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3875 {
3876         if (adev->asic_type != CHIP_ARCTURUS)
3877                 gfx_v9_0_cp_gfx_enable(adev, enable);
3878         gfx_v9_0_cp_compute_enable(adev, enable);
3879 }
3880
3881 static int gfx_v9_0_hw_init(void *handle)
3882 {
3883         int r;
3884         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3885
3886         if (!amdgpu_sriov_vf(adev))
3887                 gfx_v9_0_init_golden_registers(adev);
3888
3889         gfx_v9_0_constants_init(adev);
3890
3891         gfx_v9_0_init_tcp_config(adev);
3892
3893         r = adev->gfx.rlc.funcs->resume(adev);
3894         if (r)
3895                 return r;
3896
3897         r = gfx_v9_0_cp_resume(adev);
3898         if (r)
3899                 return r;
3900
3901         return r;
3902 }
3903
3904 static int gfx_v9_0_hw_fini(void *handle)
3905 {
3906         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3907
3908         amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
3909         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3910         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3911
3912         /* DF freeze and kcq disable will fail */
3913         if (!amdgpu_ras_intr_triggered())
3914                 /* disable KCQ to avoid CPC touch memory not valid anymore */
3915                 amdgpu_gfx_disable_kcq(adev);
3916
3917         if (amdgpu_sriov_vf(adev)) {
3918                 gfx_v9_0_cp_gfx_enable(adev, false);
3919                 /* must disable polling for SRIOV when hw finished, otherwise
3920                  * CPC engine may still keep fetching WB address which is already
3921                  * invalid after sw finished and trigger DMAR reading error in
3922                  * hypervisor side.
3923                  */
3924                 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3925                 return 0;
3926         }
3927
3928         /* Use deinitialize sequence from CAIL when unbinding device from driver,
3929          * otherwise KIQ is hanging when binding back
3930          */
3931         if (!adev->in_gpu_reset && !adev->in_suspend) {
3932                 mutex_lock(&adev->srbm_mutex);
3933                 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
3934                                 adev->gfx.kiq.ring.pipe,
3935                                 adev->gfx.kiq.ring.queue, 0);
3936                 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
3937                 soc15_grbm_select(adev, 0, 0, 0, 0);
3938                 mutex_unlock(&adev->srbm_mutex);
3939         }
3940
3941         gfx_v9_0_cp_enable(adev, false);
3942         adev->gfx.rlc.funcs->stop(adev);
3943
3944         return 0;
3945 }
3946
3947 static int gfx_v9_0_suspend(void *handle)
3948 {
3949         return gfx_v9_0_hw_fini(handle);
3950 }
3951
3952 static int gfx_v9_0_resume(void *handle)
3953 {
3954         return gfx_v9_0_hw_init(handle);
3955 }
3956
3957 static bool gfx_v9_0_is_idle(void *handle)
3958 {
3959         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3960
3961         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
3962                                 GRBM_STATUS, GUI_ACTIVE))
3963                 return false;
3964         else
3965                 return true;
3966 }
3967
3968 static int gfx_v9_0_wait_for_idle(void *handle)
3969 {
3970         unsigned i;
3971         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3972
3973         for (i = 0; i < adev->usec_timeout; i++) {
3974                 if (gfx_v9_0_is_idle(handle))
3975                         return 0;
3976                 udelay(1);
3977         }
3978         return -ETIMEDOUT;
3979 }
3980
3981 static int gfx_v9_0_soft_reset(void *handle)
3982 {
3983         u32 grbm_soft_reset = 0;
3984         u32 tmp;
3985         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3986
3987         /* GRBM_STATUS */
3988         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
3989         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
3990                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
3991                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
3992                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
3993                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
3994                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
3995                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3996                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3997                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3998                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
3999         }
4000
4001         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4002                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4003                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4004         }
4005
4006         /* GRBM_STATUS2 */
4007         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4008         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4009                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4010                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4011
4012
4013         if (grbm_soft_reset) {
4014                 /* stop the rlc */
4015                 adev->gfx.rlc.funcs->stop(adev);
4016
4017                 if (adev->asic_type != CHIP_ARCTURUS)
4018                         /* Disable GFX parsing/prefetching */
4019                         gfx_v9_0_cp_gfx_enable(adev, false);
4020
4021                 /* Disable MEC parsing/prefetching */
4022                 gfx_v9_0_cp_compute_enable(adev, false);
4023
4024                 if (grbm_soft_reset) {
4025                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4026                         tmp |= grbm_soft_reset;
4027                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4028                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4029                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4030
4031                         udelay(50);
4032
4033                         tmp &= ~grbm_soft_reset;
4034                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4035                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4036                 }
4037
4038                 /* Wait a little for things to settle down */
4039                 udelay(50);
4040         }
4041         return 0;
4042 }
4043
4044 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4045 {
4046         signed long r, cnt = 0;
4047         unsigned long flags;
4048         uint32_t seq, reg_val_offs = 0;
4049         uint64_t value = 0;
4050         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4051         struct amdgpu_ring *ring = &kiq->ring;
4052
4053         BUG_ON(!ring->funcs->emit_rreg);
4054
4055         spin_lock_irqsave(&kiq->ring_lock, flags);
4056         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
4057                 pr_err("critical bug! too many kiq readers\n");
4058                 goto failed_unlock;
4059         }
4060         amdgpu_ring_alloc(ring, 32);
4061         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4062         amdgpu_ring_write(ring, 9 |     /* src: register*/
4063                                 (5 << 8) |      /* dst: memory */
4064                                 (1 << 16) |     /* count sel */
4065                                 (1 << 20));     /* write confirm */
4066         amdgpu_ring_write(ring, 0);
4067         amdgpu_ring_write(ring, 0);
4068         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4069                                 reg_val_offs * 4));
4070         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4071                                 reg_val_offs * 4));
4072         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4073         if (r)
4074                 goto failed_undo;
4075
4076         amdgpu_ring_commit(ring);
4077         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4078
4079         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4080
4081         /* don't wait anymore for gpu reset case because this way may
4082          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4083          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4084          * never return if we keep waiting in virt_kiq_rreg, which cause
4085          * gpu_recover() hang there.
4086          *
4087          * also don't wait anymore for IRQ context
4088          * */
4089         if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
4090                 goto failed_kiq_read;
4091
4092         might_sleep();
4093         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4094                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4095                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4096         }
4097
4098         if (cnt > MAX_KIQ_REG_TRY)
4099                 goto failed_kiq_read;
4100
4101         mb();
4102         value = (uint64_t)adev->wb.wb[reg_val_offs] |
4103                 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4104         amdgpu_device_wb_free(adev, reg_val_offs);
4105         return value;
4106
4107 failed_undo:
4108         amdgpu_ring_undo(ring);
4109 failed_unlock:
4110         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4111 failed_kiq_read:
4112         if (reg_val_offs)
4113                 amdgpu_device_wb_free(adev, reg_val_offs);
4114         pr_err("failed to read gpu clock\n");
4115         return ~0;
4116 }
4117
4118 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4119 {
4120         uint64_t clock;
4121
4122         amdgpu_gfx_off_ctrl(adev, false);
4123         mutex_lock(&adev->gfx.gpu_clock_mutex);
4124         if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
4125                 clock = gfx_v9_0_kiq_read_clock(adev);
4126         } else {
4127                 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4128                 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4129                         ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4130         }
4131         mutex_unlock(&adev->gfx.gpu_clock_mutex);
4132         amdgpu_gfx_off_ctrl(adev, true);
4133         return clock;
4134 }
4135
4136 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4137                                           uint32_t vmid,
4138                                           uint32_t gds_base, uint32_t gds_size,
4139                                           uint32_t gws_base, uint32_t gws_size,
4140                                           uint32_t oa_base, uint32_t oa_size)
4141 {
4142         struct amdgpu_device *adev = ring->adev;
4143
4144         /* GDS Base */
4145         gfx_v9_0_write_data_to_reg(ring, 0, false,
4146                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4147                                    gds_base);
4148
4149         /* GDS Size */
4150         gfx_v9_0_write_data_to_reg(ring, 0, false,
4151                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4152                                    gds_size);
4153
4154         /* GWS */
4155         gfx_v9_0_write_data_to_reg(ring, 0, false,
4156                                    SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4157                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4158
4159         /* OA */
4160         gfx_v9_0_write_data_to_reg(ring, 0, false,
4161                                    SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4162                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
4163 }
4164
4165 static const u32 vgpr_init_compute_shader[] =
4166 {
4167         0xb07c0000, 0xbe8000ff,
4168         0x000000f8, 0xbf110800,
4169         0x7e000280, 0x7e020280,
4170         0x7e040280, 0x7e060280,
4171         0x7e080280, 0x7e0a0280,
4172         0x7e0c0280, 0x7e0e0280,
4173         0x80808800, 0xbe803200,
4174         0xbf84fff5, 0xbf9c0000,
4175         0xd28c0001, 0x0001007f,
4176         0xd28d0001, 0x0002027e,
4177         0x10020288, 0xb8810904,
4178         0xb7814000, 0xd1196a01,
4179         0x00000301, 0xbe800087,
4180         0xbefc00c1, 0xd89c4000,
4181         0x00020201, 0xd89cc080,
4182         0x00040401, 0x320202ff,
4183         0x00000800, 0x80808100,
4184         0xbf84fff8, 0x7e020280,
4185         0xbf810000, 0x00000000,
4186 };
4187
4188 static const u32 sgpr_init_compute_shader[] =
4189 {
4190         0xb07c0000, 0xbe8000ff,
4191         0x0000005f, 0xbee50080,
4192         0xbe812c65, 0xbe822c65,
4193         0xbe832c65, 0xbe842c65,
4194         0xbe852c65, 0xb77c0005,
4195         0x80808500, 0xbf84fff8,
4196         0xbe800080, 0xbf810000,
4197 };
4198
4199 static const u32 vgpr_init_compute_shader_arcturus[] = {
4200         0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4201         0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4202         0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4203         0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4204         0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4205         0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4206         0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4207         0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4208         0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4209         0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4210         0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4211         0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4212         0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4213         0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4214         0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4215         0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4216         0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4217         0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4218         0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4219         0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4220         0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4221         0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4222         0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4223         0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4224         0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4225         0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4226         0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4227         0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4228         0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4229         0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4230         0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4231         0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4232         0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4233         0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4234         0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4235         0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4236         0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4237         0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4238         0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4239         0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4240         0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4241         0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4242         0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4243         0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4244         0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4245         0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4246         0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4247         0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4248         0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4249         0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4250         0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4251         0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4252         0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4253         0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4254         0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4255         0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4256         0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4257         0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4258         0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4259         0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4260         0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4261         0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4262         0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4263         0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4264         0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4265         0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4266         0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4267         0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4268         0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4269         0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4270         0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4271         0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4272         0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4273         0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4274         0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4275         0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4276         0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4277         0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4278         0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4279         0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4280         0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4281         0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4282         0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4283         0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4284         0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4285         0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4286         0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4287         0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4288         0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4289         0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4290         0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4291         0xbf84fff8, 0xbf810000,
4292 };
4293
4294 /* When below register arrays changed, please update gpr_reg_size,
4295   and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4296   to cover all gfx9 ASICs */
4297 static const struct soc15_reg_entry vgpr_init_regs[] = {
4298    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4299    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4300    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4301    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4302    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4303    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4304    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4305    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4306    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4307    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4308    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4309    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4310    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4311    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4312 };
4313
4314 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4315    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4316    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4317    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4318    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4319    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4320    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4321    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4322    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4323    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4324    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4325    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4326    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4327    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4328    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4329 };
4330
4331 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4332    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4333    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4334    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4335    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4336    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4337    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4338    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4339    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4340    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4341    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4342    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4343    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4344    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4345    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4346 };
4347
4348 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4349    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4350    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4351    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4352    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4353    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4354    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4355    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4356    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4357    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4358    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4359    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4360    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4361    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4362    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4363 };
4364
4365 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4366    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4367    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4368    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4369    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4370    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4371    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4372    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4373    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4374    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4375    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4376    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4377    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4378    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4379    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4380    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4381    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4382    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4383    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4384    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4385    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4386    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4387    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4388    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4389    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4390    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4391    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4392    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4393    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4394    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4395    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4396    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4397    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4398    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4399 };
4400
4401 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4402 {
4403         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4404         int i, r;
4405
4406         /* only support when RAS is enabled */
4407         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4408                 return 0;
4409
4410         r = amdgpu_ring_alloc(ring, 7);
4411         if (r) {
4412                 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4413                         ring->name, r);
4414                 return r;
4415         }
4416
4417         WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4418         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4419
4420         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4421         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4422                                 PACKET3_DMA_DATA_DST_SEL(1) |
4423                                 PACKET3_DMA_DATA_SRC_SEL(2) |
4424                                 PACKET3_DMA_DATA_ENGINE(0)));
4425         amdgpu_ring_write(ring, 0);
4426         amdgpu_ring_write(ring, 0);
4427         amdgpu_ring_write(ring, 0);
4428         amdgpu_ring_write(ring, 0);
4429         amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4430                                 adev->gds.gds_size);
4431
4432         amdgpu_ring_commit(ring);
4433
4434         for (i = 0; i < adev->usec_timeout; i++) {
4435                 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4436                         break;
4437                 udelay(1);
4438         }
4439
4440         if (i >= adev->usec_timeout)
4441                 r = -ETIMEDOUT;
4442
4443         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4444
4445         return r;
4446 }
4447
4448 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4449 {
4450         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4451         struct amdgpu_ib ib;
4452         struct dma_fence *f = NULL;
4453         int r, i;
4454         unsigned total_size, vgpr_offset, sgpr_offset;
4455         u64 gpu_addr;
4456
4457         int compute_dim_x = adev->gfx.config.max_shader_engines *
4458                                                 adev->gfx.config.max_cu_per_sh *
4459                                                 adev->gfx.config.max_sh_per_se;
4460         int sgpr_work_group_size = 5;
4461         int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4462         int vgpr_init_shader_size;
4463         const u32 *vgpr_init_shader_ptr;
4464         const struct soc15_reg_entry *vgpr_init_regs_ptr;
4465
4466         /* only support when RAS is enabled */
4467         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4468                 return 0;
4469
4470         /* bail if the compute ring is not ready */
4471         if (!ring->sched.ready)
4472                 return 0;
4473
4474         if (adev->asic_type == CHIP_ARCTURUS) {
4475                 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4476                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4477                 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4478         } else {
4479                 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4480                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4481                 vgpr_init_regs_ptr = vgpr_init_regs;
4482         }
4483
4484         total_size =
4485                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4486         total_size +=
4487                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4488         total_size +=
4489                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4490         total_size = ALIGN(total_size, 256);
4491         vgpr_offset = total_size;
4492         total_size += ALIGN(vgpr_init_shader_size, 256);
4493         sgpr_offset = total_size;
4494         total_size += sizeof(sgpr_init_compute_shader);
4495
4496         /* allocate an indirect buffer to put the commands in */
4497         memset(&ib, 0, sizeof(ib));
4498         r = amdgpu_ib_get(adev, NULL, total_size,
4499                                         AMDGPU_IB_POOL_DIRECT, &ib);
4500         if (r) {
4501                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4502                 return r;
4503         }
4504
4505         /* load the compute shaders */
4506         for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4507                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4508
4509         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4510                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4511
4512         /* init the ib length to 0 */
4513         ib.length_dw = 0;
4514
4515         /* VGPR */
4516         /* write the register state for the compute dispatch */
4517         for (i = 0; i < gpr_reg_size; i++) {
4518                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4519                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4520                                                                 - PACKET3_SET_SH_REG_START;
4521                 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4522         }
4523         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4524         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4525         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4526         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4527                                                         - PACKET3_SET_SH_REG_START;
4528         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4529         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4530
4531         /* write dispatch packet */
4532         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4533         ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4534         ib.ptr[ib.length_dw++] = 1; /* y */
4535         ib.ptr[ib.length_dw++] = 1; /* z */
4536         ib.ptr[ib.length_dw++] =
4537                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4538
4539         /* write CS partial flush packet */
4540         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4541         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4542
4543         /* SGPR1 */
4544         /* write the register state for the compute dispatch */
4545         for (i = 0; i < gpr_reg_size; i++) {
4546                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4547                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4548                                                                 - PACKET3_SET_SH_REG_START;
4549                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4550         }
4551         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4552         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4553         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4554         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4555                                                         - PACKET3_SET_SH_REG_START;
4556         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4557         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4558
4559         /* write dispatch packet */
4560         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4561         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4562         ib.ptr[ib.length_dw++] = 1; /* y */
4563         ib.ptr[ib.length_dw++] = 1; /* z */
4564         ib.ptr[ib.length_dw++] =
4565                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4566
4567         /* write CS partial flush packet */
4568         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4569         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4570
4571         /* SGPR2 */
4572         /* write the register state for the compute dispatch */
4573         for (i = 0; i < gpr_reg_size; i++) {
4574                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4575                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4576                                                                 - PACKET3_SET_SH_REG_START;
4577                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4578         }
4579         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4580         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4581         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4582         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4583                                                         - PACKET3_SET_SH_REG_START;
4584         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4585         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4586
4587         /* write dispatch packet */
4588         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4589         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4590         ib.ptr[ib.length_dw++] = 1; /* y */
4591         ib.ptr[ib.length_dw++] = 1; /* z */
4592         ib.ptr[ib.length_dw++] =
4593                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4594
4595         /* write CS partial flush packet */
4596         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4597         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4598
4599         /* shedule the ib on the ring */
4600         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4601         if (r) {
4602                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4603                 goto fail;
4604         }
4605
4606         /* wait for the GPU to finish processing the IB */
4607         r = dma_fence_wait(f, false);
4608         if (r) {
4609                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4610                 goto fail;
4611         }
4612
4613 fail:
4614         amdgpu_ib_free(adev, &ib, NULL);
4615         dma_fence_put(f);
4616
4617         return r;
4618 }
4619
4620 static int gfx_v9_0_early_init(void *handle)
4621 {
4622         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4623
4624         if (adev->asic_type == CHIP_ARCTURUS)
4625                 adev->gfx.num_gfx_rings = 0;
4626         else
4627                 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4628         adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
4629         gfx_v9_0_set_kiq_pm4_funcs(adev);
4630         gfx_v9_0_set_ring_funcs(adev);
4631         gfx_v9_0_set_irq_funcs(adev);
4632         gfx_v9_0_set_gds_init(adev);
4633         gfx_v9_0_set_rlc_funcs(adev);
4634
4635         return 0;
4636 }
4637
4638 static int gfx_v9_0_ecc_late_init(void *handle)
4639 {
4640         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4641         int r;
4642
4643         /*
4644          * Temp workaround to fix the issue that CP firmware fails to
4645          * update read pointer when CPDMA is writing clearing operation
4646          * to GDS in suspend/resume sequence on several cards. So just
4647          * limit this operation in cold boot sequence.
4648          */
4649         if (!adev->in_suspend) {
4650                 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4651                 if (r)
4652                         return r;
4653         }
4654
4655         /* requires IBs so do in late init after IB pool is initialized */
4656         r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4657         if (r)
4658                 return r;
4659
4660         if (adev->gfx.funcs &&
4661             adev->gfx.funcs->reset_ras_error_count)
4662                 adev->gfx.funcs->reset_ras_error_count(adev);
4663
4664         r = amdgpu_gfx_ras_late_init(adev);
4665         if (r)
4666                 return r;
4667
4668         return 0;
4669 }
4670
4671 static int gfx_v9_0_late_init(void *handle)
4672 {
4673         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4674         int r;
4675
4676         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4677         if (r)
4678                 return r;
4679
4680         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4681         if (r)
4682                 return r;
4683
4684         r = gfx_v9_0_ecc_late_init(handle);
4685         if (r)
4686                 return r;
4687
4688         return 0;
4689 }
4690
4691 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4692 {
4693         uint32_t rlc_setting;
4694
4695         /* if RLC is not enabled, do nothing */
4696         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4697         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4698                 return false;
4699
4700         return true;
4701 }
4702
4703 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4704 {
4705         uint32_t data;
4706         unsigned i;
4707
4708         data = RLC_SAFE_MODE__CMD_MASK;
4709         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4710         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4711
4712         /* wait for RLC_SAFE_MODE */
4713         for (i = 0; i < adev->usec_timeout; i++) {
4714                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4715                         break;
4716                 udelay(1);
4717         }
4718 }
4719
4720 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4721 {
4722         uint32_t data;
4723
4724         data = RLC_SAFE_MODE__CMD_MASK;
4725         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4726 }
4727
4728 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4729                                                 bool enable)
4730 {
4731         amdgpu_gfx_rlc_enter_safe_mode(adev);
4732
4733         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4734                 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4735                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4736                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4737         } else {
4738                 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4739                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4740                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4741         }
4742
4743         amdgpu_gfx_rlc_exit_safe_mode(adev);
4744 }
4745
4746 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4747                                                 bool enable)
4748 {
4749         /* TODO: double check if we need to perform under safe mode */
4750         /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4751
4752         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4753                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4754         else
4755                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4756
4757         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4758                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4759         else
4760                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4761
4762         /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4763 }
4764
4765 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4766                                                       bool enable)
4767 {
4768         uint32_t data, def;
4769
4770         amdgpu_gfx_rlc_enter_safe_mode(adev);
4771
4772         /* It is disabled by HW by default */
4773         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4774                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4775                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4776
4777                 if (adev->asic_type != CHIP_VEGA12)
4778                         data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4779
4780                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4781                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4782                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4783
4784                 /* only for Vega10 & Raven1 */
4785                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4786
4787                 if (def != data)
4788                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4789
4790                 /* MGLS is a global flag to control all MGLS in GFX */
4791                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4792                         /* 2 - RLC memory Light sleep */
4793                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4794                                 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4795                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4796                                 if (def != data)
4797                                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4798                         }
4799                         /* 3 - CP memory Light sleep */
4800                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4801                                 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4802                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4803                                 if (def != data)
4804                                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4805                         }
4806                 }
4807         } else {
4808                 /* 1 - MGCG_OVERRIDE */
4809                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4810
4811                 if (adev->asic_type != CHIP_VEGA12)
4812                         data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4813
4814                 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4815                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4816                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4817                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4818
4819                 if (def != data)
4820                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4821
4822                 /* 2 - disable MGLS in RLC */
4823                 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4824                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4825                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4826                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4827                 }
4828
4829                 /* 3 - disable MGLS in CP */
4830                 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4831                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4832                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4833                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4834                 }
4835         }
4836
4837         amdgpu_gfx_rlc_exit_safe_mode(adev);
4838 }
4839
4840 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4841                                            bool enable)
4842 {
4843         uint32_t data, def;
4844
4845         if (adev->asic_type == CHIP_ARCTURUS)
4846                 return;
4847
4848         amdgpu_gfx_rlc_enter_safe_mode(adev);
4849
4850         /* Enable 3D CGCG/CGLS */
4851         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
4852                 /* write cmd to clear cgcg/cgls ov */
4853                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4854                 /* unset CGCG override */
4855                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4856                 /* update CGCG and CGLS override bits */
4857                 if (def != data)
4858                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4859
4860                 /* enable 3Dcgcg FSM(0x0000363f) */
4861                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4862
4863                 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4864                         RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4865                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4866                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4867                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4868                 if (def != data)
4869                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4870
4871                 /* set IDLE_POLL_COUNT(0x00900100) */
4872                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4873                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4874                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4875                 if (def != data)
4876                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4877         } else {
4878                 /* Disable CGCG/CGLS */
4879                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4880                 /* disable cgcg, cgls should be disabled */
4881                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4882                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4883                 /* disable cgcg and cgls in FSM */
4884                 if (def != data)
4885                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4886         }
4887
4888         amdgpu_gfx_rlc_exit_safe_mode(adev);
4889 }
4890
4891 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4892                                                       bool enable)
4893 {
4894         uint32_t def, data;
4895
4896         amdgpu_gfx_rlc_enter_safe_mode(adev);
4897
4898         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4899                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4900                 /* unset CGCG override */
4901                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4902                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4903                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4904                 else
4905                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4906                 /* update CGCG and CGLS override bits */
4907                 if (def != data)
4908                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4909
4910                 /* enable cgcg FSM(0x0000363F) */
4911                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4912
4913                 if (adev->asic_type == CHIP_ARCTURUS)
4914                         data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4915                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4916                 else
4917                         data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4918                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4919                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4920                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4921                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4922                 if (def != data)
4923                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4924
4925                 /* set IDLE_POLL_COUNT(0x00900100) */
4926                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4927                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4928                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4929                 if (def != data)
4930                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4931         } else {
4932                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4933                 /* reset CGCG/CGLS bits */
4934                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
4935                 /* disable cgcg and cgls in FSM */
4936                 if (def != data)
4937                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4938         }
4939
4940         amdgpu_gfx_rlc_exit_safe_mode(adev);
4941 }
4942
4943 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4944                                             bool enable)
4945 {
4946         if (enable) {
4947                 /* CGCG/CGLS should be enabled after MGCG/MGLS
4948                  * ===  MGCG + MGLS ===
4949                  */
4950                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4951                 /* ===  CGCG /CGLS for GFX 3D Only === */
4952                 gfx_v9_0_update_3d_clock_gating(adev, enable);
4953                 /* ===  CGCG + CGLS === */
4954                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4955         } else {
4956                 /* CGCG/CGLS should be disabled before MGCG/MGLS
4957                  * ===  CGCG + CGLS ===
4958                  */
4959                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4960                 /* ===  CGCG /CGLS for GFX 3D Only === */
4961                 gfx_v9_0_update_3d_clock_gating(adev, enable);
4962                 /* ===  MGCG + MGLS === */
4963                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4964         }
4965         return 0;
4966 }
4967
4968 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
4969 {
4970         u32 reg, data;
4971
4972         reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
4973         if (amdgpu_sriov_is_pp_one_vf(adev))
4974                 data = RREG32_NO_KIQ(reg);
4975         else
4976                 data = RREG32(reg);
4977
4978         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
4979         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
4980
4981         if (amdgpu_sriov_is_pp_one_vf(adev))
4982                 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
4983         else
4984                 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
4985 }
4986
4987 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
4988                                         uint32_t offset,
4989                                         struct soc15_reg_rlcg *entries, int arr_size)
4990 {
4991         int i;
4992         uint32_t reg;
4993
4994         if (!entries)
4995                 return false;
4996
4997         for (i = 0; i < arr_size; i++) {
4998                 const struct soc15_reg_rlcg *entry;
4999
5000                 entry = &entries[i];
5001                 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5002                 if (offset == reg)
5003                         return true;
5004         }
5005
5006         return false;
5007 }
5008
5009 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5010 {
5011         return gfx_v9_0_check_rlcg_range(adev, offset,
5012                                         (void *)rlcg_access_gc_9_0,
5013                                         ARRAY_SIZE(rlcg_access_gc_9_0));
5014 }
5015
5016 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5017         .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5018         .set_safe_mode = gfx_v9_0_set_safe_mode,
5019         .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5020         .init = gfx_v9_0_rlc_init,
5021         .get_csb_size = gfx_v9_0_get_csb_size,
5022         .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5023         .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5024         .resume = gfx_v9_0_rlc_resume,
5025         .stop = gfx_v9_0_rlc_stop,
5026         .reset = gfx_v9_0_rlc_reset,
5027         .start = gfx_v9_0_rlc_start,
5028         .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5029         .rlcg_wreg = gfx_v9_0_rlcg_wreg,
5030         .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5031 };
5032
5033 static int gfx_v9_0_set_powergating_state(void *handle,
5034                                           enum amd_powergating_state state)
5035 {
5036         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5037         bool enable = (state == AMD_PG_STATE_GATE);
5038
5039         switch (adev->asic_type) {
5040         case CHIP_RAVEN:
5041         case CHIP_RENOIR:
5042                 if (!enable)
5043                         amdgpu_gfx_off_ctrl(adev, false);
5044
5045                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5046                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5047                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5048                 } else {
5049                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5050                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5051                 }
5052
5053                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5054                         gfx_v9_0_enable_cp_power_gating(adev, true);
5055                 else
5056                         gfx_v9_0_enable_cp_power_gating(adev, false);
5057
5058                 /* update gfx cgpg state */
5059                 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5060
5061                 /* update mgcg state */
5062                 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5063
5064                 if (enable)
5065                         amdgpu_gfx_off_ctrl(adev, true);
5066                 break;
5067         case CHIP_VEGA12:
5068                 amdgpu_gfx_off_ctrl(adev, enable);
5069                 break;
5070         default:
5071                 break;
5072         }
5073
5074         return 0;
5075 }
5076
5077 static int gfx_v9_0_set_clockgating_state(void *handle,
5078                                           enum amd_clockgating_state state)
5079 {
5080         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5081
5082         if (amdgpu_sriov_vf(adev))
5083                 return 0;
5084
5085         switch (adev->asic_type) {
5086         case CHIP_VEGA10:
5087         case CHIP_VEGA12:
5088         case CHIP_VEGA20:
5089         case CHIP_RAVEN:
5090         case CHIP_ARCTURUS:
5091         case CHIP_RENOIR:
5092                 gfx_v9_0_update_gfx_clock_gating(adev,
5093                                                  state == AMD_CG_STATE_GATE);
5094                 break;
5095         default:
5096                 break;
5097         }
5098         return 0;
5099 }
5100
5101 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
5102 {
5103         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5104         int data;
5105
5106         if (amdgpu_sriov_vf(adev))
5107                 *flags = 0;
5108
5109         /* AMD_CG_SUPPORT_GFX_MGCG */
5110         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5111         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5112                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5113
5114         /* AMD_CG_SUPPORT_GFX_CGCG */
5115         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5116         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5117                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5118
5119         /* AMD_CG_SUPPORT_GFX_CGLS */
5120         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5121                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5122
5123         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5124         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5125         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5126                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5127
5128         /* AMD_CG_SUPPORT_GFX_CP_LS */
5129         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5130         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5131                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5132
5133         if (adev->asic_type != CHIP_ARCTURUS) {
5134                 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5135                 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5136                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5137                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5138
5139                 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5140                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5141                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5142         }
5143 }
5144
5145 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5146 {
5147         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
5148 }
5149
5150 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5151 {
5152         struct amdgpu_device *adev = ring->adev;
5153         u64 wptr;
5154
5155         /* XXX check if swapping is necessary on BE */
5156         if (ring->use_doorbell) {
5157                 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
5158         } else {
5159                 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5160                 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5161         }
5162
5163         return wptr;
5164 }
5165
5166 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5167 {
5168         struct amdgpu_device *adev = ring->adev;
5169
5170         if (ring->use_doorbell) {
5171                 /* XXX check if swapping is necessary on BE */
5172                 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5173                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5174         } else {
5175                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5176                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5177         }
5178 }
5179
5180 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5181 {
5182         struct amdgpu_device *adev = ring->adev;
5183         u32 ref_and_mask, reg_mem_engine;
5184         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5185
5186         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5187                 switch (ring->me) {
5188                 case 1:
5189                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5190                         break;
5191                 case 2:
5192                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5193                         break;
5194                 default:
5195                         return;
5196                 }
5197                 reg_mem_engine = 0;
5198         } else {
5199                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5200                 reg_mem_engine = 1; /* pfp */
5201         }
5202
5203         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5204                               adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5205                               adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5206                               ref_and_mask, ref_and_mask, 0x20);
5207 }
5208
5209 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5210                                         struct amdgpu_job *job,
5211                                         struct amdgpu_ib *ib,
5212                                         uint32_t flags)
5213 {
5214         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5215         u32 header, control = 0;
5216
5217         if (ib->flags & AMDGPU_IB_FLAG_CE)
5218                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5219         else
5220                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5221
5222         control |= ib->length_dw | (vmid << 24);
5223
5224         if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5225                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5226
5227                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5228                         gfx_v9_0_ring_emit_de_meta(ring);
5229         }
5230
5231         amdgpu_ring_write(ring, header);
5232         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5233         amdgpu_ring_write(ring,
5234 #ifdef __BIG_ENDIAN
5235                 (2 << 0) |
5236 #endif
5237                 lower_32_bits(ib->gpu_addr));
5238         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5239         amdgpu_ring_write(ring, control);
5240 }
5241
5242 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5243                                           struct amdgpu_job *job,
5244                                           struct amdgpu_ib *ib,
5245                                           uint32_t flags)
5246 {
5247         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5248         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5249
5250         /* Currently, there is a high possibility to get wave ID mismatch
5251          * between ME and GDS, leading to a hw deadlock, because ME generates
5252          * different wave IDs than the GDS expects. This situation happens
5253          * randomly when at least 5 compute pipes use GDS ordered append.
5254          * The wave IDs generated by ME are also wrong after suspend/resume.
5255          * Those are probably bugs somewhere else in the kernel driver.
5256          *
5257          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5258          * GDS to 0 for this ring (me/pipe).
5259          */
5260         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5261                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5262                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5263                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5264         }
5265
5266         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5267         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5268         amdgpu_ring_write(ring,
5269 #ifdef __BIG_ENDIAN
5270                                 (2 << 0) |
5271 #endif
5272                                 lower_32_bits(ib->gpu_addr));
5273         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5274         amdgpu_ring_write(ring, control);
5275 }
5276
5277 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5278                                      u64 seq, unsigned flags)
5279 {
5280         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5281         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5282         bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5283
5284         /* RELEASE_MEM - flush caches, send int */
5285         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5286         amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
5287                                                EOP_TC_NC_ACTION_EN) :
5288                                               (EOP_TCL1_ACTION_EN |
5289                                                EOP_TC_ACTION_EN |
5290                                                EOP_TC_WB_ACTION_EN |
5291                                                EOP_TC_MD_ACTION_EN)) |
5292                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5293                                  EVENT_INDEX(5)));
5294         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5295
5296         /*
5297          * the address should be Qword aligned if 64bit write, Dword
5298          * aligned if only send 32bit data low (discard data high)
5299          */
5300         if (write64bit)
5301                 BUG_ON(addr & 0x7);
5302         else
5303                 BUG_ON(addr & 0x3);
5304         amdgpu_ring_write(ring, lower_32_bits(addr));
5305         amdgpu_ring_write(ring, upper_32_bits(addr));
5306         amdgpu_ring_write(ring, lower_32_bits(seq));
5307         amdgpu_ring_write(ring, upper_32_bits(seq));
5308         amdgpu_ring_write(ring, 0);
5309 }
5310
5311 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5312 {
5313         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5314         uint32_t seq = ring->fence_drv.sync_seq;
5315         uint64_t addr = ring->fence_drv.gpu_addr;
5316
5317         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5318                               lower_32_bits(addr), upper_32_bits(addr),
5319                               seq, 0xffffffff, 4);
5320 }
5321
5322 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5323                                         unsigned vmid, uint64_t pd_addr)
5324 {
5325         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5326
5327         /* compute doesn't have PFP */
5328         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5329                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5330                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5331                 amdgpu_ring_write(ring, 0x0);
5332         }
5333 }
5334
5335 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5336 {
5337         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
5338 }
5339
5340 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5341 {
5342         u64 wptr;
5343
5344         /* XXX check if swapping is necessary on BE */
5345         if (ring->use_doorbell)
5346                 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
5347         else
5348                 BUG();
5349         return wptr;
5350 }
5351
5352 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5353 {
5354         struct amdgpu_device *adev = ring->adev;
5355
5356         /* XXX check if swapping is necessary on BE */
5357         if (ring->use_doorbell) {
5358                 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5359                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5360         } else{
5361                 BUG(); /* only DOORBELL method supported on gfx9 now */
5362         }
5363 }
5364
5365 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5366                                          u64 seq, unsigned int flags)
5367 {
5368         struct amdgpu_device *adev = ring->adev;
5369
5370         /* we only allocate 32bit for each seq wb address */
5371         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5372
5373         /* write fence seq to the "addr" */
5374         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5375         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5376                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5377         amdgpu_ring_write(ring, lower_32_bits(addr));
5378         amdgpu_ring_write(ring, upper_32_bits(addr));
5379         amdgpu_ring_write(ring, lower_32_bits(seq));
5380
5381         if (flags & AMDGPU_FENCE_FLAG_INT) {
5382                 /* set register to trigger INT */
5383                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5384                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5385                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5386                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5387                 amdgpu_ring_write(ring, 0);
5388                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5389         }
5390 }
5391
5392 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5393 {
5394         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5395         amdgpu_ring_write(ring, 0);
5396 }
5397
5398 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
5399 {
5400         struct v9_ce_ib_state ce_payload = {0};
5401         uint64_t csa_addr;
5402         int cnt;
5403
5404         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5405         csa_addr = amdgpu_csa_vaddr(ring->adev);
5406
5407         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5408         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5409                                  WRITE_DATA_DST_SEL(8) |
5410                                  WR_CONFIRM) |
5411                                  WRITE_DATA_CACHE_POLICY(0));
5412         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5413         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5414         amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
5415 }
5416
5417 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
5418 {
5419         struct v9_de_ib_state de_payload = {0};
5420         uint64_t csa_addr, gds_addr;
5421         int cnt;
5422
5423         csa_addr = amdgpu_csa_vaddr(ring->adev);
5424         gds_addr = csa_addr + 4096;
5425         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5426         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5427
5428         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5429         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5430         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5431                                  WRITE_DATA_DST_SEL(8) |
5432                                  WR_CONFIRM) |
5433                                  WRITE_DATA_CACHE_POLICY(0));
5434         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5435         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5436         amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
5437 }
5438
5439 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5440                                    bool secure)
5441 {
5442         uint32_t v = secure ? FRAME_TMZ : 0;
5443
5444         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5445         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5446 }
5447
5448 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5449 {
5450         uint32_t dw2 = 0;
5451
5452         if (amdgpu_sriov_vf(ring->adev))
5453                 gfx_v9_0_ring_emit_ce_meta(ring);
5454
5455         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5456         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5457                 /* set load_global_config & load_global_uconfig */
5458                 dw2 |= 0x8001;
5459                 /* set load_cs_sh_regs */
5460                 dw2 |= 0x01000000;
5461                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5462                 dw2 |= 0x10002;
5463
5464                 /* set load_ce_ram if preamble presented */
5465                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5466                         dw2 |= 0x10000000;
5467         } else {
5468                 /* still load_ce_ram if this is the first time preamble presented
5469                  * although there is no context switch happens.
5470                  */
5471                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5472                         dw2 |= 0x10000000;
5473         }
5474
5475         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5476         amdgpu_ring_write(ring, dw2);
5477         amdgpu_ring_write(ring, 0);
5478 }
5479
5480 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5481 {
5482         unsigned ret;
5483         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5484         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5485         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5486         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5487         ret = ring->wptr & ring->buf_mask;
5488         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5489         return ret;
5490 }
5491
5492 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5493 {
5494         unsigned cur;
5495         BUG_ON(offset > ring->buf_mask);
5496         BUG_ON(ring->ring[offset] != 0x55aa55aa);
5497
5498         cur = (ring->wptr & ring->buf_mask) - 1;
5499         if (likely(cur > offset))
5500                 ring->ring[offset] = cur - offset;
5501         else
5502                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5503 }
5504
5505 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5506                                     uint32_t reg_val_offs)
5507 {
5508         struct amdgpu_device *adev = ring->adev;
5509
5510         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5511         amdgpu_ring_write(ring, 0 |     /* src: register*/
5512                                 (5 << 8) |      /* dst: memory */
5513                                 (1 << 20));     /* write confirm */
5514         amdgpu_ring_write(ring, reg);
5515         amdgpu_ring_write(ring, 0);
5516         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5517                                 reg_val_offs * 4));
5518         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5519                                 reg_val_offs * 4));
5520 }
5521
5522 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5523                                     uint32_t val)
5524 {
5525         uint32_t cmd = 0;
5526
5527         switch (ring->funcs->type) {
5528         case AMDGPU_RING_TYPE_GFX:
5529                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5530                 break;
5531         case AMDGPU_RING_TYPE_KIQ:
5532                 cmd = (1 << 16); /* no inc addr */
5533                 break;
5534         default:
5535                 cmd = WR_CONFIRM;
5536                 break;
5537         }
5538         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5539         amdgpu_ring_write(ring, cmd);
5540         amdgpu_ring_write(ring, reg);
5541         amdgpu_ring_write(ring, 0);
5542         amdgpu_ring_write(ring, val);
5543 }
5544
5545 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5546                                         uint32_t val, uint32_t mask)
5547 {
5548         gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5549 }
5550
5551 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5552                                                   uint32_t reg0, uint32_t reg1,
5553                                                   uint32_t ref, uint32_t mask)
5554 {
5555         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5556         struct amdgpu_device *adev = ring->adev;
5557         bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5558                 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5559
5560         if (fw_version_ok)
5561                 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5562                                       ref, mask, 0x20);
5563         else
5564                 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5565                                                            ref, mask);
5566 }
5567
5568 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5569 {
5570         struct amdgpu_device *adev = ring->adev;
5571         uint32_t value = 0;
5572
5573         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5574         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5575         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5576         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5577         WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5578 }
5579
5580 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5581                                                  enum amdgpu_interrupt_state state)
5582 {
5583         switch (state) {
5584         case AMDGPU_IRQ_STATE_DISABLE:
5585         case AMDGPU_IRQ_STATE_ENABLE:
5586                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5587                                TIME_STAMP_INT_ENABLE,
5588                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5589                 break;
5590         default:
5591                 break;
5592         }
5593 }
5594
5595 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5596                                                      int me, int pipe,
5597                                                      enum amdgpu_interrupt_state state)
5598 {
5599         u32 mec_int_cntl, mec_int_cntl_reg;
5600
5601         /*
5602          * amdgpu controls only the first MEC. That's why this function only
5603          * handles the setting of interrupts for this specific MEC. All other
5604          * pipes' interrupts are set by amdkfd.
5605          */
5606
5607         if (me == 1) {
5608                 switch (pipe) {
5609                 case 0:
5610                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5611                         break;
5612                 case 1:
5613                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5614                         break;
5615                 case 2:
5616                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5617                         break;
5618                 case 3:
5619                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5620                         break;
5621                 default:
5622                         DRM_DEBUG("invalid pipe %d\n", pipe);
5623                         return;
5624                 }
5625         } else {
5626                 DRM_DEBUG("invalid me %d\n", me);
5627                 return;
5628         }
5629
5630         switch (state) {
5631         case AMDGPU_IRQ_STATE_DISABLE:
5632                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5633                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5634                                              TIME_STAMP_INT_ENABLE, 0);
5635                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5636                 break;
5637         case AMDGPU_IRQ_STATE_ENABLE:
5638                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5639                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5640                                              TIME_STAMP_INT_ENABLE, 1);
5641                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5642                 break;
5643         default:
5644                 break;
5645         }
5646 }
5647
5648 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5649                                              struct amdgpu_irq_src *source,
5650                                              unsigned type,
5651                                              enum amdgpu_interrupt_state state)
5652 {
5653         switch (state) {
5654         case AMDGPU_IRQ_STATE_DISABLE:
5655         case AMDGPU_IRQ_STATE_ENABLE:
5656                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5657                                PRIV_REG_INT_ENABLE,
5658                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5659                 break;
5660         default:
5661                 break;
5662         }
5663
5664         return 0;
5665 }
5666
5667 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5668                                               struct amdgpu_irq_src *source,
5669                                               unsigned type,
5670                                               enum amdgpu_interrupt_state state)
5671 {
5672         switch (state) {
5673         case AMDGPU_IRQ_STATE_DISABLE:
5674         case AMDGPU_IRQ_STATE_ENABLE:
5675                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5676                                PRIV_INSTR_INT_ENABLE,
5677                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5678         default:
5679                 break;
5680         }
5681
5682         return 0;
5683 }
5684
5685 #define ENABLE_ECC_ON_ME_PIPE(me, pipe)                         \
5686         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5687                         CP_ECC_ERROR_INT_ENABLE, 1)
5688
5689 #define DISABLE_ECC_ON_ME_PIPE(me, pipe)                        \
5690         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5691                         CP_ECC_ERROR_INT_ENABLE, 0)
5692
5693 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5694                                               struct amdgpu_irq_src *source,
5695                                               unsigned type,
5696                                               enum amdgpu_interrupt_state state)
5697 {
5698         switch (state) {
5699         case AMDGPU_IRQ_STATE_DISABLE:
5700                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5701                                 CP_ECC_ERROR_INT_ENABLE, 0);
5702                 DISABLE_ECC_ON_ME_PIPE(1, 0);
5703                 DISABLE_ECC_ON_ME_PIPE(1, 1);
5704                 DISABLE_ECC_ON_ME_PIPE(1, 2);
5705                 DISABLE_ECC_ON_ME_PIPE(1, 3);
5706                 break;
5707
5708         case AMDGPU_IRQ_STATE_ENABLE:
5709                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5710                                 CP_ECC_ERROR_INT_ENABLE, 1);
5711                 ENABLE_ECC_ON_ME_PIPE(1, 0);
5712                 ENABLE_ECC_ON_ME_PIPE(1, 1);
5713                 ENABLE_ECC_ON_ME_PIPE(1, 2);
5714                 ENABLE_ECC_ON_ME_PIPE(1, 3);
5715                 break;
5716         default:
5717                 break;
5718         }
5719
5720         return 0;
5721 }
5722
5723
5724 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5725                                             struct amdgpu_irq_src *src,
5726                                             unsigned type,
5727                                             enum amdgpu_interrupt_state state)
5728 {
5729         switch (type) {
5730         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5731                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5732                 break;
5733         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5734                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5735                 break;
5736         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5737                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5738                 break;
5739         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5740                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5741                 break;
5742         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5743                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5744                 break;
5745         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5746                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5747                 break;
5748         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5749                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5750                 break;
5751         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5752                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5753                 break;
5754         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5755                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5756                 break;
5757         default:
5758                 break;
5759         }
5760         return 0;
5761 }
5762
5763 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5764                             struct amdgpu_irq_src *source,
5765                             struct amdgpu_iv_entry *entry)
5766 {
5767         int i;
5768         u8 me_id, pipe_id, queue_id;
5769         struct amdgpu_ring *ring;
5770
5771         DRM_DEBUG("IH: CP EOP\n");
5772         me_id = (entry->ring_id & 0x0c) >> 2;
5773         pipe_id = (entry->ring_id & 0x03) >> 0;
5774         queue_id = (entry->ring_id & 0x70) >> 4;
5775
5776         switch (me_id) {
5777         case 0:
5778                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5779                 break;
5780         case 1:
5781         case 2:
5782                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5783                         ring = &adev->gfx.compute_ring[i];
5784                         /* Per-queue interrupt is supported for MEC starting from VI.
5785                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
5786                           */
5787                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5788                                 amdgpu_fence_process(ring);
5789                 }
5790                 break;
5791         }
5792         return 0;
5793 }
5794
5795 static void gfx_v9_0_fault(struct amdgpu_device *adev,
5796                            struct amdgpu_iv_entry *entry)
5797 {
5798         u8 me_id, pipe_id, queue_id;
5799         struct amdgpu_ring *ring;
5800         int i;
5801
5802         me_id = (entry->ring_id & 0x0c) >> 2;
5803         pipe_id = (entry->ring_id & 0x03) >> 0;
5804         queue_id = (entry->ring_id & 0x70) >> 4;
5805
5806         switch (me_id) {
5807         case 0:
5808                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5809                 break;
5810         case 1:
5811         case 2:
5812                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5813                         ring = &adev->gfx.compute_ring[i];
5814                         if (ring->me == me_id && ring->pipe == pipe_id &&
5815                             ring->queue == queue_id)
5816                                 drm_sched_fault(&ring->sched);
5817                 }
5818                 break;
5819         }
5820 }
5821
5822 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5823                                  struct amdgpu_irq_src *source,
5824                                  struct amdgpu_iv_entry *entry)
5825 {
5826         DRM_ERROR("Illegal register access in command stream\n");
5827         gfx_v9_0_fault(adev, entry);
5828         return 0;
5829 }
5830
5831 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5832                                   struct amdgpu_irq_src *source,
5833                                   struct amdgpu_iv_entry *entry)
5834 {
5835         DRM_ERROR("Illegal instruction in command stream\n");
5836         gfx_v9_0_fault(adev, entry);
5837         return 0;
5838 }
5839
5840
5841 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5842         { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5843           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5844           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5845         },
5846         { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5847           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5848           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5849         },
5850         { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5851           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5852           0, 0
5853         },
5854         { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5855           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5856           0, 0
5857         },
5858         { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5859           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5860           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5861         },
5862         { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5863           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5864           0, 0
5865         },
5866         { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5867           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
5868           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
5869         },
5870         { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
5871           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
5872           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
5873         },
5874         { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
5875           SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
5876           0, 0
5877         },
5878         { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
5879           SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
5880           0, 0
5881         },
5882         { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
5883           SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
5884           0, 0
5885         },
5886         { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5887           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
5888           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
5889         },
5890         { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5891           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
5892           0, 0
5893         },
5894         { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5895           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
5896           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
5897         },
5898         { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
5899           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5900           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
5901           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
5902         },
5903         { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
5904           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5905           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
5906           0, 0
5907         },
5908         { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
5909           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5910           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
5911           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
5912         },
5913         { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
5914           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5915           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
5916           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
5917         },
5918         { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
5919           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5920           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
5921           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
5922         },
5923         { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
5924           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5925           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
5926           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
5927         },
5928         { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
5929           SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
5930           0, 0
5931         },
5932         { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5933           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
5934           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
5935         },
5936         { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5937           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
5938           0, 0
5939         },
5940         { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5941           SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
5942           0, 0
5943         },
5944         { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5945           SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
5946           0, 0
5947         },
5948         { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5949           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
5950           0, 0
5951         },
5952         { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5953           SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
5954           0, 0
5955         },
5956         { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5957           SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
5958           0, 0
5959         },
5960         { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5961           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
5962           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
5963         },
5964         { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5965           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
5966           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
5967         },
5968         { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5969           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
5970           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
5971         },
5972         { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5973           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
5974           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
5975         },
5976         { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5977           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
5978           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
5979         },
5980         { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5981           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
5982           0, 0
5983         },
5984         { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5985           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
5986           0, 0
5987         },
5988         { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5989           SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
5990           0, 0
5991         },
5992         { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5993           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
5994           0, 0
5995         },
5996         { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5997           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
5998           0, 0
5999         },
6000         { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6001           SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6002           0, 0
6003         },
6004         { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6005           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6006           0, 0
6007         },
6008         { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6009           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6010           0, 0
6011         },
6012         { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6013           SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6014           0, 0
6015         },
6016         { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6017           SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6018           0, 0
6019         },
6020         { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6021           SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6022           0, 0
6023         },
6024         { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6025           SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6026           0, 0
6027         },
6028         { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6029           SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6030           0, 0
6031         },
6032         { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6033           SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6034           0, 0
6035         },
6036         { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6037           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6038           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6039         },
6040         { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6041           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6042           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6043         },
6044         { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6045           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6046           0, 0
6047         },
6048         { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6049           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6050           0, 0
6051         },
6052         { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6053           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6054           0, 0
6055         },
6056         { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6057           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6058           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6059         },
6060         { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6061           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6062           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6063         },
6064         { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6065           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6066           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6067         },
6068         { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6069           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6070           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6071         },
6072         { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6073           SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6074           0, 0
6075         },
6076         { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6077           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6078           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6079         },
6080         { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6081           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6082           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6083         },
6084         { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6085           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6086           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6087         },
6088         { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6089           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6090           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6091         },
6092         { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6093           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6094           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6095         },
6096         { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6097           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6098           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6099         },
6100         { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6101           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6102           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6103         },
6104         { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6105           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6106           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6107         },
6108         { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6109           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6110           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6111         },
6112         { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6113           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6114           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6115         },
6116         { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6117           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6118           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6119         },
6120         { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6121           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6122           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6123         },
6124         { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6125           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6126           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6127         },
6128         { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6129           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6130           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6131         },
6132         { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6133           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6134           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6135         },
6136         { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6137           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6138           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6139         },
6140         { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6141           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6142           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6143         },
6144         { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6145           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6146           0, 0
6147         },
6148         { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6149           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6150           0, 0
6151         },
6152         { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6153           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6154           0, 0
6155         },
6156         { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6157           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6158           0, 0
6159         },
6160         { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6161           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6162           0, 0
6163         },
6164         { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6165           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6166           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6167         },
6168         { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6169           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6170           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6171         },
6172         { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6173           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6174           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6175         },
6176         { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6177           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6178           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6179         },
6180         { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6181           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6182           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6183         },
6184         { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6185           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6186           0, 0
6187         },
6188         { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6189           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6190           0, 0
6191         },
6192         { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6193           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6194           0, 0
6195         },
6196         { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6197           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6198           0, 0
6199         },
6200         { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6201           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6202           0, 0
6203         },
6204         { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6205           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6206           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6207         },
6208         { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6209           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6210           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6211         },
6212         { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6213           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6214           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6215         },
6216         { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6217           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6218           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6219         },
6220         { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6221           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6222           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6223         },
6224         { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6225           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6226           0, 0
6227         },
6228         { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6229           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6230           0, 0
6231         },
6232         { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6233           SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6234           0, 0
6235         },
6236         { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6237           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6238           0, 0
6239         },
6240         { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6241           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6242           0, 0
6243         },
6244         { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6245           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6246           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6247         },
6248         { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6249           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6250           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6251         },
6252         { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6253           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6254           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6255         },
6256         { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6257           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6258           0, 0
6259         },
6260         { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6261           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6262           0, 0
6263         },
6264         { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6265           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6266           0, 0
6267         },
6268         { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6269           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6270           0, 0
6271         },
6272         { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6273           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6274           0, 0
6275         },
6276         { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6277           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6278           0, 0
6279         }
6280 };
6281
6282 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6283                                      void *inject_if)
6284 {
6285         struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6286         int ret;
6287         struct ta_ras_trigger_error_input block_info = { 0 };
6288
6289         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6290                 return -EINVAL;
6291
6292         if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6293                 return -EINVAL;
6294
6295         if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6296                 return -EPERM;
6297
6298         if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6299               info->head.type)) {
6300                 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6301                         ras_gfx_subblocks[info->head.sub_block_index].name,
6302                         info->head.type);
6303                 return -EPERM;
6304         }
6305
6306         if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6307               info->head.type)) {
6308                 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6309                         ras_gfx_subblocks[info->head.sub_block_index].name,
6310                         info->head.type);
6311                 return -EPERM;
6312         }
6313
6314         block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6315         block_info.sub_block_index =
6316                 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6317         block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6318         block_info.address = info->address;
6319         block_info.value = info->value;
6320
6321         mutex_lock(&adev->grbm_idx_mutex);
6322         ret = psp_ras_trigger_error(&adev->psp, &block_info);
6323         mutex_unlock(&adev->grbm_idx_mutex);
6324
6325         return ret;
6326 }
6327
6328 static const char *vml2_mems[] = {
6329         "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6330         "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6331         "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6332         "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6333         "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6334         "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6335         "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6336         "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6337         "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6338         "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6339         "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6340         "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6341         "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6342         "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6343         "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6344         "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6345 };
6346
6347 static const char *vml2_walker_mems[] = {
6348         "UTC_VML2_CACHE_PDE0_MEM0",
6349         "UTC_VML2_CACHE_PDE0_MEM1",
6350         "UTC_VML2_CACHE_PDE1_MEM0",
6351         "UTC_VML2_CACHE_PDE1_MEM1",
6352         "UTC_VML2_CACHE_PDE2_MEM0",
6353         "UTC_VML2_CACHE_PDE2_MEM1",
6354         "UTC_VML2_RDIF_LOG_FIFO",
6355 };
6356
6357 static const char *atc_l2_cache_2m_mems[] = {
6358         "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6359         "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6360         "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6361         "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6362 };
6363
6364 static const char *atc_l2_cache_4k_mems[] = {
6365         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6366         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6367         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6368         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6369         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6370         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6371         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6372         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6373         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6374         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6375         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6376         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6377         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6378         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6379         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6380         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6381         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6382         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6383         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6384         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6385         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6386         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6387         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6388         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6389         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6390         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6391         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6392         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6393         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6394         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6395         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6396         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6397 };
6398
6399 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6400                                          struct ras_err_data *err_data)
6401 {
6402         uint32_t i, data;
6403         uint32_t sec_count, ded_count;
6404
6405         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6406         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6407         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6408         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6409         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6410         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6411         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6412         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6413
6414         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6415                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6416                 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6417
6418                 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6419                 if (sec_count) {
6420                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6421                                 "SEC %d\n", i, vml2_mems[i], sec_count);
6422                         err_data->ce_count += sec_count;
6423                 }
6424
6425                 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6426                 if (ded_count) {
6427                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6428                                 "DED %d\n", i, vml2_mems[i], ded_count);
6429                         err_data->ue_count += ded_count;
6430                 }
6431         }
6432
6433         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6434                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6435                 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6436
6437                 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6438                                                 SEC_COUNT);
6439                 if (sec_count) {
6440                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6441                                 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6442                         err_data->ce_count += sec_count;
6443                 }
6444
6445                 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6446                                                 DED_COUNT);
6447                 if (ded_count) {
6448                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6449                                 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6450                         err_data->ue_count += ded_count;
6451                 }
6452         }
6453
6454         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6455                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6456                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6457
6458                 sec_count = (data & 0x00006000L) >> 0xd;
6459                 if (sec_count) {
6460                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6461                                 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6462                                 sec_count);
6463                         err_data->ce_count += sec_count;
6464                 }
6465         }
6466
6467         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6468                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6469                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6470
6471                 sec_count = (data & 0x00006000L) >> 0xd;
6472                 if (sec_count) {
6473                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6474                                 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6475                                 sec_count);
6476                         err_data->ce_count += sec_count;
6477                 }
6478
6479                 ded_count = (data & 0x00018000L) >> 0xf;
6480                 if (ded_count) {
6481                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6482                                 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6483                                 ded_count);
6484                         err_data->ue_count += ded_count;
6485                 }
6486         }
6487
6488         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6489         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6490         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6491         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6492
6493         return 0;
6494 }
6495
6496 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6497         const struct soc15_reg_entry *reg,
6498         uint32_t se_id, uint32_t inst_id, uint32_t value,
6499         uint32_t *sec_count, uint32_t *ded_count)
6500 {
6501         uint32_t i;
6502         uint32_t sec_cnt, ded_cnt;
6503
6504         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6505                 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6506                         gfx_v9_0_ras_fields[i].seg != reg->seg ||
6507                         gfx_v9_0_ras_fields[i].inst != reg->inst)
6508                         continue;
6509
6510                 sec_cnt = (value &
6511                                 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6512                                 gfx_v9_0_ras_fields[i].sec_count_shift;
6513                 if (sec_cnt) {
6514                         dev_info(adev->dev, "GFX SubBlock %s, "
6515                                 "Instance[%d][%d], SEC %d\n",
6516                                 gfx_v9_0_ras_fields[i].name,
6517                                 se_id, inst_id,
6518                                 sec_cnt);
6519                         *sec_count += sec_cnt;
6520                 }
6521
6522                 ded_cnt = (value &
6523                                 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6524                                 gfx_v9_0_ras_fields[i].ded_count_shift;
6525                 if (ded_cnt) {
6526                         dev_info(adev->dev, "GFX SubBlock %s, "
6527                                 "Instance[%d][%d], DED %d\n",
6528                                 gfx_v9_0_ras_fields[i].name,
6529                                 se_id, inst_id,
6530                                 ded_cnt);
6531                         *ded_count += ded_cnt;
6532                 }
6533         }
6534
6535         return 0;
6536 }
6537
6538 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6539 {
6540         int i, j, k;
6541
6542         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6543                 return;
6544
6545         /* read back registers to clear the counters */
6546         mutex_lock(&adev->grbm_idx_mutex);
6547         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6548                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6549                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6550                                 gfx_v9_0_select_se_sh(adev, j, 0x0, k);
6551                                 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6552                         }
6553                 }
6554         }
6555         WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6556         mutex_unlock(&adev->grbm_idx_mutex);
6557
6558         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6559         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6560         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6561         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6562         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6563         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6564         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6565         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6566
6567         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6568                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6569                 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6570         }
6571
6572         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6573                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6574                 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6575         }
6576
6577         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6578                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6579                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6580         }
6581
6582         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6583                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6584                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6585         }
6586
6587         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6588         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6589         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6590         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6591 }
6592
6593 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6594                                           void *ras_error_status)
6595 {
6596         struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6597         uint32_t sec_count = 0, ded_count = 0;
6598         uint32_t i, j, k;
6599         uint32_t reg_value;
6600
6601         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6602                 return -EINVAL;
6603
6604         err_data->ue_count = 0;
6605         err_data->ce_count = 0;
6606
6607         mutex_lock(&adev->grbm_idx_mutex);
6608
6609         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6610                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6611                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6612                                 gfx_v9_0_select_se_sh(adev, j, 0, k);
6613                                 reg_value =
6614                                         RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6615                                 if (reg_value)
6616                                         gfx_v9_0_ras_error_count(adev,
6617                                                 &gfx_v9_0_edc_counter_regs[i],
6618                                                 j, k, reg_value,
6619                                                 &sec_count, &ded_count);
6620                         }
6621                 }
6622         }
6623
6624         err_data->ce_count += sec_count;
6625         err_data->ue_count += ded_count;
6626
6627         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6628         mutex_unlock(&adev->grbm_idx_mutex);
6629
6630         gfx_v9_0_query_utc_edc_status(adev, err_data);
6631
6632         return 0;
6633 }
6634
6635 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6636 {
6637         const unsigned int cp_coher_cntl =
6638                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6639                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6640                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6641                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6642                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6643
6644         /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
6645         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6646         amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
6647         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6648         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6649         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6650         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6651         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6652 }
6653
6654 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6655         .name = "gfx_v9_0",
6656         .early_init = gfx_v9_0_early_init,
6657         .late_init = gfx_v9_0_late_init,
6658         .sw_init = gfx_v9_0_sw_init,
6659         .sw_fini = gfx_v9_0_sw_fini,
6660         .hw_init = gfx_v9_0_hw_init,
6661         .hw_fini = gfx_v9_0_hw_fini,
6662         .suspend = gfx_v9_0_suspend,
6663         .resume = gfx_v9_0_resume,
6664         .is_idle = gfx_v9_0_is_idle,
6665         .wait_for_idle = gfx_v9_0_wait_for_idle,
6666         .soft_reset = gfx_v9_0_soft_reset,
6667         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6668         .set_powergating_state = gfx_v9_0_set_powergating_state,
6669         .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6670 };
6671
6672 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6673         .type = AMDGPU_RING_TYPE_GFX,
6674         .align_mask = 0xff,
6675         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6676         .support_64bit_ptrs = true,
6677         .vmhub = AMDGPU_GFXHUB_0,
6678         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6679         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6680         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6681         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6682                 5 +  /* COND_EXEC */
6683                 7 +  /* PIPELINE_SYNC */
6684                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6685                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6686                 2 + /* VM_FLUSH */
6687                 8 +  /* FENCE for VM_FLUSH */
6688                 20 + /* GDS switch */
6689                 4 + /* double SWITCH_BUFFER,
6690                        the first COND_EXEC jump to the place just
6691                            prior to this double SWITCH_BUFFER  */
6692                 5 + /* COND_EXEC */
6693                 7 +      /*     HDP_flush */
6694                 4 +      /*     VGT_flush */
6695                 14 + /* CE_META */
6696                 31 + /* DE_META */
6697                 3 + /* CNTX_CTRL */
6698                 5 + /* HDP_INVL */
6699                 8 + 8 + /* FENCE x2 */
6700                 2 + /* SWITCH_BUFFER */
6701                 7, /* gfx_v9_0_emit_mem_sync */
6702         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6703         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6704         .emit_fence = gfx_v9_0_ring_emit_fence,
6705         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6706         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6707         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6708         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6709         .test_ring = gfx_v9_0_ring_test_ring,
6710         .test_ib = gfx_v9_0_ring_test_ib,
6711         .insert_nop = amdgpu_ring_insert_nop,
6712         .pad_ib = amdgpu_ring_generic_pad_ib,
6713         .emit_switch_buffer = gfx_v9_ring_emit_sb,
6714         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6715         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6716         .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6717         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6718         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6719         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6720         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6721         .soft_recovery = gfx_v9_0_ring_soft_recovery,
6722         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6723 };
6724
6725 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6726         .type = AMDGPU_RING_TYPE_COMPUTE,
6727         .align_mask = 0xff,
6728         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6729         .support_64bit_ptrs = true,
6730         .vmhub = AMDGPU_GFXHUB_0,
6731         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6732         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6733         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6734         .emit_frame_size =
6735                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6736                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6737                 5 + /* hdp invalidate */
6738                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6739                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6740                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6741                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6742                 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6743                 7, /* gfx_v9_0_emit_mem_sync */
6744         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6745         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6746         .emit_fence = gfx_v9_0_ring_emit_fence,
6747         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6748         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6749         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6750         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6751         .test_ring = gfx_v9_0_ring_test_ring,
6752         .test_ib = gfx_v9_0_ring_test_ib,
6753         .insert_nop = amdgpu_ring_insert_nop,
6754         .pad_ib = amdgpu_ring_generic_pad_ib,
6755         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6756         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6757         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6758         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6759 };
6760
6761 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6762         .type = AMDGPU_RING_TYPE_KIQ,
6763         .align_mask = 0xff,
6764         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6765         .support_64bit_ptrs = true,
6766         .vmhub = AMDGPU_GFXHUB_0,
6767         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6768         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6769         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6770         .emit_frame_size =
6771                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6772                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6773                 5 + /* hdp invalidate */
6774                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6775                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6776                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6777                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6778                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6779         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6780         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6781         .test_ring = gfx_v9_0_ring_test_ring,
6782         .insert_nop = amdgpu_ring_insert_nop,
6783         .pad_ib = amdgpu_ring_generic_pad_ib,
6784         .emit_rreg = gfx_v9_0_ring_emit_rreg,
6785         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6786         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6787         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6788 };
6789
6790 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6791 {
6792         int i;
6793
6794         adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
6795
6796         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6797                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
6798
6799         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6800                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
6801 }
6802
6803 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
6804         .set = gfx_v9_0_set_eop_interrupt_state,
6805         .process = gfx_v9_0_eop_irq,
6806 };
6807
6808 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
6809         .set = gfx_v9_0_set_priv_reg_fault_state,
6810         .process = gfx_v9_0_priv_reg_irq,
6811 };
6812
6813 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
6814         .set = gfx_v9_0_set_priv_inst_fault_state,
6815         .process = gfx_v9_0_priv_inst_irq,
6816 };
6817
6818 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
6819         .set = gfx_v9_0_set_cp_ecc_error_state,
6820         .process = amdgpu_gfx_cp_ecc_error_irq,
6821 };
6822
6823
6824 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
6825 {
6826         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6827         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
6828
6829         adev->gfx.priv_reg_irq.num_types = 1;
6830         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
6831
6832         adev->gfx.priv_inst_irq.num_types = 1;
6833         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
6834
6835         adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
6836         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
6837 }
6838
6839 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
6840 {
6841         switch (adev->asic_type) {
6842         case CHIP_VEGA10:
6843         case CHIP_VEGA12:
6844         case CHIP_VEGA20:
6845         case CHIP_RAVEN:
6846         case CHIP_ARCTURUS:
6847         case CHIP_RENOIR:
6848                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
6849                 break;
6850         default:
6851                 break;
6852         }
6853 }
6854
6855 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
6856 {
6857         /* init asci gds info */
6858         switch (adev->asic_type) {
6859         case CHIP_VEGA10:
6860         case CHIP_VEGA12:
6861         case CHIP_VEGA20:
6862                 adev->gds.gds_size = 0x10000;
6863                 break;
6864         case CHIP_RAVEN:
6865         case CHIP_ARCTURUS:
6866                 adev->gds.gds_size = 0x1000;
6867                 break;
6868         default:
6869                 adev->gds.gds_size = 0x10000;
6870                 break;
6871         }
6872
6873         switch (adev->asic_type) {
6874         case CHIP_VEGA10:
6875         case CHIP_VEGA20:
6876                 adev->gds.gds_compute_max_wave_id = 0x7ff;
6877                 break;
6878         case CHIP_VEGA12:
6879                 adev->gds.gds_compute_max_wave_id = 0x27f;
6880                 break;
6881         case CHIP_RAVEN:
6882                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
6883                         adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
6884                 else
6885                         adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
6886                 break;
6887         case CHIP_ARCTURUS:
6888                 adev->gds.gds_compute_max_wave_id = 0xfff;
6889                 break;
6890         default:
6891                 /* this really depends on the chip */
6892                 adev->gds.gds_compute_max_wave_id = 0x7ff;
6893                 break;
6894         }
6895
6896         adev->gds.gws_size = 64;
6897         adev->gds.oa_size = 16;
6898 }
6899
6900 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
6901                                                  u32 bitmap)
6902 {
6903         u32 data;
6904
6905         if (!bitmap)
6906                 return;
6907
6908         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
6909         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
6910
6911         WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
6912 }
6913
6914 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
6915 {
6916         u32 data, mask;
6917
6918         data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
6919         data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
6920
6921         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
6922         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
6923
6924         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
6925
6926         return (~data) & mask;
6927 }
6928
6929 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
6930                                  struct amdgpu_cu_info *cu_info)
6931 {
6932         int i, j, k, counter, active_cu_number = 0;
6933         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
6934         unsigned disable_masks[4 * 4];
6935
6936         if (!adev || !cu_info)
6937                 return -EINVAL;
6938
6939         /*
6940          * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
6941          */
6942         if (adev->gfx.config.max_shader_engines *
6943                 adev->gfx.config.max_sh_per_se > 16)
6944                 return -EINVAL;
6945
6946         amdgpu_gfx_parse_disable_cu(disable_masks,
6947                                     adev->gfx.config.max_shader_engines,
6948                                     adev->gfx.config.max_sh_per_se);
6949
6950         mutex_lock(&adev->grbm_idx_mutex);
6951         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
6952                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
6953                         mask = 1;
6954                         ao_bitmap = 0;
6955                         counter = 0;
6956                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
6957                         gfx_v9_0_set_user_cu_inactive_bitmap(
6958                                 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
6959                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
6960
6961                         /*
6962                          * The bitmap(and ao_cu_bitmap) in cu_info structure is
6963                          * 4x4 size array, and it's usually suitable for Vega
6964                          * ASICs which has 4*2 SE/SH layout.
6965                          * But for Arcturus, SE/SH layout is changed to 8*1.
6966                          * To mostly reduce the impact, we make it compatible
6967                          * with current bitmap array as below:
6968                          *    SE4,SH0 --> bitmap[0][1]
6969                          *    SE5,SH0 --> bitmap[1][1]
6970                          *    SE6,SH0 --> bitmap[2][1]
6971                          *    SE7,SH0 --> bitmap[3][1]
6972                          */
6973                         cu_info->bitmap[i % 4][j + i / 4] = bitmap;
6974
6975                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
6976                                 if (bitmap & mask) {
6977                                         if (counter < adev->gfx.config.max_cu_per_sh)
6978                                                 ao_bitmap |= mask;
6979                                         counter ++;
6980                                 }
6981                                 mask <<= 1;
6982                         }
6983                         active_cu_number += counter;
6984                         if (i < 2 && j < 2)
6985                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
6986                         cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
6987                 }
6988         }
6989         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6990         mutex_unlock(&adev->grbm_idx_mutex);
6991
6992         cu_info->number = active_cu_number;
6993         cu_info->ao_cu_mask = ao_cu_mask;
6994         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
6995
6996         return 0;
6997 }
6998
6999 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7000 {
7001         .type = AMD_IP_BLOCK_TYPE_GFX,
7002         .major = 9,
7003         .minor = 0,
7004         .rev = 0,
7005         .funcs = &gfx_v9_0_ip_funcs,
7006 };