drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "amdgpu_ring.h"
  33 #include "vi.h"
  34 #include "vi_structs.h"
  35 #include "vid.h"
  36 #include "amdgpu_ucode.h"
  37 #include "amdgpu_atombios.h"
  38 #include "atombios_i2c.h"
  39 #include "clearstate_vi.h"
  40
  41 #include "gmc/gmc_8_2_d.h"
  42 #include "gmc/gmc_8_2_sh_mask.h"
  43
  44 #include "oss/oss_3_0_d.h"
  45 #include "oss/oss_3_0_sh_mask.h"
  46
  47 #include "bif/bif_5_0_d.h"
  48 #include "bif/bif_5_0_sh_mask.h"
  49 #include "gca/gfx_8_0_d.h"
  50 #include "gca/gfx_8_0_enum.h"
  51 #include "gca/gfx_8_0_sh_mask.h"
  52
  53 #include "dce/dce_10_0_d.h"
  54 #include "dce/dce_10_0_sh_mask.h"
  55
  56 #include "smu/smu_7_1_3_d.h"
  57
  58 #include "ivsrcid/ivsrcid_vislands30.h"
  59
  60 #define GFX8_NUM_GFX_RINGS     1
  61 #define GFX8_MEC_HPD_SIZE 4096
  62
  63 #define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x22010001
  64 #define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x22010001
  65 #define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x22011002
  66 #define TONGA_GB_ADDR_CONFIG_GOLDEN 0x22011003
  67
  68 #define ARRAY_MODE(x)                                   ((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT)
  69 #define PIPE_CONFIG(x)                                  ((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT)
  70 #define TILE_SPLIT(x)                                   ((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT)
  71 #define MICRO_TILE_MODE_NEW(x)                          ((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT)
  72 #define SAMPLE_SPLIT(x)                                 ((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT)
  73 #define BANK_WIDTH(x)                                   ((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT)
  74 #define BANK_HEIGHT(x)                                  ((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT)
  75 #define MACRO_TILE_ASPECT(x)                            ((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT)
  76 #define NUM_BANKS(x)                                    ((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT)
  77
  78 #define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK            0x00000001L
  79 #define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK            0x00000002L
  80 #define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK           0x00000004L
  81 #define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK           0x00000008L
  82 #define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK           0x00000010L
  83 #define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK           0x00000020L
  84
  85 /* BPM SERDES CMD */
  86 #define SET_BPM_SERDES_CMD    1
  87 #define CLE_BPM_SERDES_CMD    0
  88
  89 /* BPM Register Address*/
  90 enum {
  91         BPM_REG_CGLS_EN = 0,        /* Enable/Disable CGLS */
  92         BPM_REG_CGLS_ON,            /* ON/OFF CGLS: shall be controlled by RLC FW */
  93         BPM_REG_CGCG_OVERRIDE,      /* Set/Clear CGCG Override */
  94         BPM_REG_MGCG_OVERRIDE,      /* Set/Clear MGCG Override */
  95         BPM_REG_FGCG_OVERRIDE,      /* Set/Clear FGCG Override */
  96         BPM_REG_FGCG_MAX
  97 };
  98
  99 #define RLC_FormatDirectRegListLength        14
 100
 101 MODULE_FIRMWARE("amdgpu/carrizo_ce.bin");
 102 MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin");
 103 MODULE_FIRMWARE("amdgpu/carrizo_me.bin");
 104 MODULE_FIRMWARE("amdgpu/carrizo_mec.bin");
 105 MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin");
 106 MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin");
 107
 108 MODULE_FIRMWARE("amdgpu/stoney_ce.bin");
 109 MODULE_FIRMWARE("amdgpu/stoney_pfp.bin");
 110 MODULE_FIRMWARE("amdgpu/stoney_me.bin");
 111 MODULE_FIRMWARE("amdgpu/stoney_mec.bin");
 112 MODULE_FIRMWARE("amdgpu/stoney_rlc.bin");
 113
 114 MODULE_FIRMWARE("amdgpu/tonga_ce.bin");
 115 MODULE_FIRMWARE("amdgpu/tonga_pfp.bin");
 116 MODULE_FIRMWARE("amdgpu/tonga_me.bin");
 117 MODULE_FIRMWARE("amdgpu/tonga_mec.bin");
 118 MODULE_FIRMWARE("amdgpu/tonga_mec2.bin");
 119 MODULE_FIRMWARE("amdgpu/tonga_rlc.bin");
 120
 121 MODULE_FIRMWARE("amdgpu/topaz_ce.bin");
 122 MODULE_FIRMWARE("amdgpu/topaz_pfp.bin");
 123 MODULE_FIRMWARE("amdgpu/topaz_me.bin");
 124 MODULE_FIRMWARE("amdgpu/topaz_mec.bin");
 125 MODULE_FIRMWARE("amdgpu/topaz_rlc.bin");
 126
 127 MODULE_FIRMWARE("amdgpu/fiji_ce.bin");
 128 MODULE_FIRMWARE("amdgpu/fiji_pfp.bin");
 129 MODULE_FIRMWARE("amdgpu/fiji_me.bin");
 130 MODULE_FIRMWARE("amdgpu/fiji_mec.bin");
 131 MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");
 132 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");
 133
 134 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin");
 135 MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin");
 136 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin");
 137 MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin");
 138 MODULE_FIRMWARE("amdgpu/polaris10_me.bin");
 139 MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin");
 140 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin");
 141 MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin");
 142 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin");
 143 MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin");
 144 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin");
 145
 146 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");
 147 MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin");
 148 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");
 149 MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin");
 150 MODULE_FIRMWARE("amdgpu/polaris11_me.bin");
 151 MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin");
 152 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");
 153 MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin");
 154 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");
 155 MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin");
 156 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");
 157
 158 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin");
 159 MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin");
 160 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin");
 161 MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin");
 162 MODULE_FIRMWARE("amdgpu/polaris12_me.bin");
 163 MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin");
 164 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin");
 165 MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin");
 166 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin");
 167 MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin");
 168 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin");
 169
 170 MODULE_FIRMWARE("amdgpu/vegam_ce.bin");
 171 MODULE_FIRMWARE("amdgpu/vegam_pfp.bin");
 172 MODULE_FIRMWARE("amdgpu/vegam_me.bin");
 173 MODULE_FIRMWARE("amdgpu/vegam_mec.bin");
 174 MODULE_FIRMWARE("amdgpu/vegam_mec2.bin");
 175 MODULE_FIRMWARE("amdgpu/vegam_rlc.bin");
 176
 177 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
 178 {
 179         {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
 180         {mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1},
 181         {mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2},
 182         {mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3},
 183         {mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4},
 184         {mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5},
 185         {mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6},
 186         {mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7},
 187         {mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8},
 188         {mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9},
 189         {mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10},
 190         {mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11},
 191         {mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12},
 192         {mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13},
 193         {mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14},
 194         {mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15}
 195 };
 196
 197 static const u32 golden_settings_tonga_a11[] =
 198 {
 199         mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
 200         mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
 201         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 202         mmGB_GPU_ID, 0x0000000f, 0x00000000,
 203         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 204         mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
 205         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 206         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
 207         mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 208         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 209         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 210         mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 211         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
 212         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
 213         mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
 214         mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
 215 };
 216
 217 static const u32 tonga_golden_common_all[] =
 218 {
 219         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 220         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
 221         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
 222         mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 223         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 224         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 225         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 226         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
 227 };
 228
 229 static const u32 tonga_mgcg_cgcg_init[] =
 230 {
 231         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
 232         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 233         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 234         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 235         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 236         mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
 237         mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
 238         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 239         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 240         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 241         mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
 242         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 243         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 244         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 245         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 246         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 247         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 248         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 249         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 250         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 251         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 252         mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
 253         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 254         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 255         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 256         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 257         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 258         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 259         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 260         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 261         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 262         mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 263         mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 264         mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
 265         mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 266         mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 267         mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 268         mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 269         mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
 270         mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 271         mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 272         mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 273         mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 274         mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
 275         mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 276         mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 277         mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 278         mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 279         mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
 280         mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 281         mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 282         mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 283         mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 284         mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
 285         mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 286         mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 287         mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 288         mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 289         mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
 290         mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 291         mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 292         mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 293         mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 294         mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
 295         mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 296         mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 297         mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 298         mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 299         mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
 300         mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 301         mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 302         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
 303         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 304         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
 305         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 306 };
 307
 308 static const u32 golden_settings_vegam_a11[] =
 309 {
 310         mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
 311         mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000,
 312         mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
 313         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 314         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 315         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 316         mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a,
 317         mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e,
 318         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 319         mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
 320         mmSQ_CONFIG, 0x07f80000, 0x01180000,
 321         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 322         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 323         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 324         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 325         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
 326         mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
 327 };
 328
 329 static const u32 vegam_golden_common_all[] =
 330 {
 331         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 332         mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 333         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 334         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 335         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 336         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 337 };
 338
 339 static const u32 golden_settings_polaris11_a11[] =
 340 {
 341         mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208,
 342         mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
 343         mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
 344         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 345         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 346         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 347         mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
 348         mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
 349         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 350         mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
 351         mmSQ_CONFIG, 0x07f80000, 0x01180000,
 352         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 353         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 354         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
 355         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 356         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 357         mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
 358 };
 359
 360 static const u32 polaris11_golden_common_all[] =
 361 {
 362         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 363         mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002,
 364         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 365         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 366         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 367         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 368 };
 369
 370 static const u32 golden_settings_polaris10_a11[] =
 371 {
 372         mmATC_MISC_CG, 0x000c0fc0, 0x000c0200,
 373         mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
 374         mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
 375         mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
 376         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 377         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 378         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 379         mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
 380         mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a,
 381         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 382         mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
 383         mmSQ_CONFIG, 0x07f80000, 0x07180000,
 384         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 385         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 386         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 387         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 388         mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
 389 };
 390
 391 static const u32 polaris10_golden_common_all[] =
 392 {
 393         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 394         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
 395         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
 396         mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 397         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 398         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 399         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 400         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 401 };
 402
 403 static const u32 fiji_golden_common_all[] =
 404 {
 405         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 406         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
 407         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
 408         mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 409         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 410         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 411         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 412         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 413         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 414         mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
 415 };
 416
 417 static const u32 golden_settings_fiji_a10[] =
 418 {
 419         mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
 420         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 421         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 422         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 423         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 424         mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 425         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 426         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 427         mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 428         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
 429         mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
 430 };
 431
 432 static const u32 fiji_mgcg_cgcg_init[] =
 433 {
 434         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
 435         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 436         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 437         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 438         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 439         mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
 440         mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
 441         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 442         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 443         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 444         mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
 445         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 446         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 447         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 448         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 449         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 450         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 451         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 452         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 453         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 454         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 455         mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
 456         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 457         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 458         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 459         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 460         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 461         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 462         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 463         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 464         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 465         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
 466         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 467         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
 468         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 469 };
 470
 471 static const u32 golden_settings_iceland_a11[] =
 472 {
 473         mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
 474         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 475         mmDB_DEBUG3, 0xc0000000, 0xc0000000,
 476         mmGB_GPU_ID, 0x0000000f, 0x00000000,
 477         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 478         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 479         mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002,
 480         mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
 481         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
 482         mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 483         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 484         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 485         mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 486         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
 487         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 488         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010,
 489 };
 490
 491 static const u32 iceland_golden_common_all[] =
 492 {
 493         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 494         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
 495         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
 496         mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
 497         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 498         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 499         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 500         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
 501 };
 502
 503 static const u32 iceland_mgcg_cgcg_init[] =
 504 {
 505         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
 506         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 507         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 508         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 509         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100,
 510         mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100,
 511         mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100,
 512         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 513         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 514         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 515         mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
 516         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 517         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 518         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 519         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 520         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 521         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 522         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 523         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 524         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 525         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 526         mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
 527         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100,
 528         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 529         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 530         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 531         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 532         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 533         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 534         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 535         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 536         mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 537         mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 538         mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
 539         mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 540         mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 541         mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 542         mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 543         mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
 544         mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 545         mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 546         mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 547         mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 548         mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
 549         mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 550         mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 551         mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 552         mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 553         mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
 554         mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 555         mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 556         mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 557         mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 558         mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
 559         mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 560         mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 561         mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 562         mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 563         mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
 564         mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 565         mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 566         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
 567         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 568         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
 569 };
 570
 571 static const u32 cz_golden_settings_a11[] =
 572 {
 573         mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
 574         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 575         mmGB_GPU_ID, 0x0000000f, 0x00000000,
 576         mmPA_SC_ENHANCE, 0xffffffff, 0x00000001,
 577         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 578         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
 579         mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 580         mmTA_CNTL_AUX, 0x000f000f, 0x00010000,
 581         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 582         mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 583         mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3,
 584         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302
 585 };
 586
 587 static const u32 cz_golden_common_all[] =
 588 {
 589         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 590         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
 591         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
 592         mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
 593         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 594         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 595         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 596         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
 597 };
 598
 599 static const u32 cz_mgcg_cgcg_init[] =
 600 {
 601         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
 602         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 603         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 604         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 605         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 606         mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
 607         mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100,
 608         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 609         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 610         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 611         mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
 612         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 613         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 614         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 615         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 616         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 617         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 618         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 619         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 620         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 621         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 622         mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
 623         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 624         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 625         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 626         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 627         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 628         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 629         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 630         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 631         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 632         mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 633         mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 634         mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
 635         mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 636         mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 637         mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 638         mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 639         mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
 640         mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 641         mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 642         mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 643         mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 644         mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
 645         mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 646         mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 647         mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 648         mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 649         mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
 650         mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 651         mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 652         mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 653         mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 654         mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
 655         mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 656         mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 657         mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 658         mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 659         mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
 660         mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 661         mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 662         mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 663         mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 664         mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
 665         mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 666         mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 667         mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 668         mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 669         mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
 670         mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 671         mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 672         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
 673         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 674         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 675         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 676 };
 677
 678 static const u32 stoney_golden_settings_a11[] =
 679 {
 680         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 681         mmGB_GPU_ID, 0x0000000f, 0x00000000,
 682         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 683         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 684         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 685         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 686         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 687         mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 688         mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1,
 689         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010,
 690 };
 691
 692 static const u32 stoney_golden_common_all[] =
 693 {
 694         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 695         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000,
 696         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
 697         mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001,
 698         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 699         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 700         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 701         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 702 };
 703
 704 static const u32 stoney_mgcg_cgcg_init[] =
 705 {
 706         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 707         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 708         mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
 709         mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
 710         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 711 };
 712
 713
 714 static const char * const sq_edc_source_names[] = {
 715         "SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred",
 716         "SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch",
 717         "SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return",
 718         "SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR",
 719         "SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS",
 720         "SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS",
 721         "SQ_EDC_INFO_SOURCE_TA: EDC source is TA",
 722 };
 723
 724 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev);
 725 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev);
 726 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);
 727 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);
 728 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);
 729 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);
 730 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring);
 731 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring);
 732
 733 #define CG_ACLK_CNTL__ACLK_DIVIDER_MASK                    0x0000007fL
 734 #define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT                  0x00000000L
 735
 736 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
 737 {
 738         uint32_t data;
 739
 740         switch (adev->asic_type) {
 741         case CHIP_TOPAZ:
 742                 amdgpu_device_program_register_sequence(adev,
 743                                                         iceland_mgcg_cgcg_init,
 744                                                         ARRAY_SIZE(iceland_mgcg_cgcg_init));
 745                 amdgpu_device_program_register_sequence(adev,
 746                                                         golden_settings_iceland_a11,
 747                                                         ARRAY_SIZE(golden_settings_iceland_a11));
 748                 amdgpu_device_program_register_sequence(adev,
 749                                                         iceland_golden_common_all,
 750                                                         ARRAY_SIZE(iceland_golden_common_all));
 751                 break;
 752         case CHIP_FIJI:
 753                 amdgpu_device_program_register_sequence(adev,
 754                                                         fiji_mgcg_cgcg_init,
 755                                                         ARRAY_SIZE(fiji_mgcg_cgcg_init));
 756                 amdgpu_device_program_register_sequence(adev,
 757                                                         golden_settings_fiji_a10,
 758                                                         ARRAY_SIZE(golden_settings_fiji_a10));
 759                 amdgpu_device_program_register_sequence(adev,
 760                                                         fiji_golden_common_all,
 761                                                         ARRAY_SIZE(fiji_golden_common_all));
 762                 break;
 763
 764         case CHIP_TONGA:
 765                 amdgpu_device_program_register_sequence(adev,
 766                                                         tonga_mgcg_cgcg_init,
 767                                                         ARRAY_SIZE(tonga_mgcg_cgcg_init));
 768                 amdgpu_device_program_register_sequence(adev,
 769                                                         golden_settings_tonga_a11,
 770                                                         ARRAY_SIZE(golden_settings_tonga_a11));
 771                 amdgpu_device_program_register_sequence(adev,
 772                                                         tonga_golden_common_all,
 773                                                         ARRAY_SIZE(tonga_golden_common_all));
 774                 break;
 775         case CHIP_VEGAM:
 776                 amdgpu_device_program_register_sequence(adev,
 777                                                         golden_settings_vegam_a11,
 778                                                         ARRAY_SIZE(golden_settings_vegam_a11));
 779                 amdgpu_device_program_register_sequence(adev,
 780                                                         vegam_golden_common_all,
 781                                                         ARRAY_SIZE(vegam_golden_common_all));
 782                 break;
 783         case CHIP_POLARIS11:
 784         case CHIP_POLARIS12:
 785                 amdgpu_device_program_register_sequence(adev,
 786                                                         golden_settings_polaris11_a11,
 787                                                         ARRAY_SIZE(golden_settings_polaris11_a11));
 788                 amdgpu_device_program_register_sequence(adev,
 789                                                         polaris11_golden_common_all,
 790                                                         ARRAY_SIZE(polaris11_golden_common_all));
 791                 break;
 792         case CHIP_POLARIS10:
 793                 amdgpu_device_program_register_sequence(adev,
 794                                                         golden_settings_polaris10_a11,
 795                                                         ARRAY_SIZE(golden_settings_polaris10_a11));
 796                 amdgpu_device_program_register_sequence(adev,
 797                                                         polaris10_golden_common_all,
 798                                                         ARRAY_SIZE(polaris10_golden_common_all));
 799                 data = RREG32_SMC(ixCG_ACLK_CNTL);
 800                 data &= ~CG_ACLK_CNTL__ACLK_DIVIDER_MASK;
 801                 data |= 0x18 << CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT;
 802                 WREG32_SMC(ixCG_ACLK_CNTL, data);
 803                 if ((adev->pdev->device == 0x67DF) && (adev->pdev->revision == 0xc7) &&
 804                     ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) ||
 805                      (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) ||
 806                      (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1680))) {
 807                         amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD);
 808                         amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0);
 809                 }
 810                 break;
 811         case CHIP_CARRIZO:
 812                 amdgpu_device_program_register_sequence(adev,
 813                                                         cz_mgcg_cgcg_init,
 814                                                         ARRAY_SIZE(cz_mgcg_cgcg_init));
 815                 amdgpu_device_program_register_sequence(adev,
 816                                                         cz_golden_settings_a11,
 817                                                         ARRAY_SIZE(cz_golden_settings_a11));
 818                 amdgpu_device_program_register_sequence(adev,
 819                                                         cz_golden_common_all,
 820                                                         ARRAY_SIZE(cz_golden_common_all));
 821                 break;
 822         case CHIP_STONEY:
 823                 amdgpu_device_program_register_sequence(adev,
 824                                                         stoney_mgcg_cgcg_init,
 825                                                         ARRAY_SIZE(stoney_mgcg_cgcg_init));
 826                 amdgpu_device_program_register_sequence(adev,
 827                                                         stoney_golden_settings_a11,
 828                                                         ARRAY_SIZE(stoney_golden_settings_a11));
 829                 amdgpu_device_program_register_sequence(adev,
 830                                                         stoney_golden_common_all,
 831                                                         ARRAY_SIZE(stoney_golden_common_all));
 832                 break;
 833         default:
 834                 break;
 835         }
 836 }
 837
 838 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
 839 {
 840         struct amdgpu_device *adev = ring->adev;
 841         uint32_t tmp = 0;
 842         unsigned i;
 843         int r;
 844
 845         WREG32(mmSCRATCH_REG0, 0xCAFEDEAD);
 846         r = amdgpu_ring_alloc(ring, 3);
 847         if (r)
 848                 return r;
 849
 850         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 851         amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START);
 852         amdgpu_ring_write(ring, 0xDEADBEEF);
 853         amdgpu_ring_commit(ring);
 854
 855         for (i = 0; i < adev->usec_timeout; i++) {
 856                 tmp = RREG32(mmSCRATCH_REG0);
 857                 if (tmp == 0xDEADBEEF)
 858                         break;
 859                 udelay(1);
 860         }
 861
 862         if (i >= adev->usec_timeout)
 863                 r = -ETIMEDOUT;
 864
 865         return r;
 866 }
 867
 868 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 869 {
 870         struct amdgpu_device *adev = ring->adev;
 871         struct amdgpu_ib ib;
 872         struct dma_fence *f = NULL;
 873
 874         unsigned int index;
 875         uint64_t gpu_addr;
 876         uint32_t tmp;
 877         long r;
 878
 879         r = amdgpu_device_wb_get(adev, &index);
 880         if (r)
 881                 return r;
 882
 883         gpu_addr = adev->wb.gpu_addr + (index * 4);
 884         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 885         memset(&ib, 0, sizeof(ib));
 886         r = amdgpu_ib_get(adev, NULL, 16,
 887                                         AMDGPU_IB_POOL_DIRECT, &ib);
 888         if (r)
 889                 goto err1;
 890
 891         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
 892         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
 893         ib.ptr[2] = lower_32_bits(gpu_addr);
 894         ib.ptr[3] = upper_32_bits(gpu_addr);
 895         ib.ptr[4] = 0xDEADBEEF;
 896         ib.length_dw = 5;
 897
 898         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 899         if (r)
 900                 goto err2;
 901
 902         r = dma_fence_wait_timeout(f, false, timeout);
 903         if (r == 0) {
 904                 r = -ETIMEDOUT;
 905                 goto err2;
 906         } else if (r < 0) {
 907                 goto err2;
 908         }
 909
 910         tmp = adev->wb.wb[index];
 911         if (tmp == 0xDEADBEEF)
 912                 r = 0;
 913         else
 914                 r = -EINVAL;
 915
 916 err2:
 917         amdgpu_ib_free(adev, &ib, NULL);
 918         dma_fence_put(f);
 919 err1:
 920         amdgpu_device_wb_free(adev, index);
 921         return r;
 922 }
 923
 924
 925 static void gfx_v8_0_free_microcode(struct amdgpu_device *adev)
 926 {
 927         release_firmware(adev->gfx.pfp_fw);
 928         adev->gfx.pfp_fw = NULL;
 929         release_firmware(adev->gfx.me_fw);
 930         adev->gfx.me_fw = NULL;
 931         release_firmware(adev->gfx.ce_fw);
 932         adev->gfx.ce_fw = NULL;
 933         release_firmware(adev->gfx.rlc_fw);
 934         adev->gfx.rlc_fw = NULL;
 935         release_firmware(adev->gfx.mec_fw);
 936         adev->gfx.mec_fw = NULL;
 937         if ((adev->asic_type != CHIP_STONEY) &&
 938             (adev->asic_type != CHIP_TOPAZ))
 939                 release_firmware(adev->gfx.mec2_fw);
 940         adev->gfx.mec2_fw = NULL;
 941
 942         kfree(adev->gfx.rlc.register_list_format);
 943 }
 944
 945 static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
 946 {
 947         const char *chip_name;
 948         char fw_name[30];
 949         int err;
 950         struct amdgpu_firmware_info *info = NULL;
 951         const struct common_firmware_header *header = NULL;
 952         const struct gfx_firmware_header_v1_0 *cp_hdr;
 953         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 954         unsigned int *tmp = NULL, i;
 955
 956         DRM_DEBUG("\n");
 957
 958         switch (adev->asic_type) {
 959         case CHIP_TOPAZ:
 960                 chip_name = "topaz";
 961                 break;
 962         case CHIP_TONGA:
 963                 chip_name = "tonga";
 964                 break;
 965         case CHIP_CARRIZO:
 966                 chip_name = "carrizo";
 967                 break;
 968         case CHIP_FIJI:
 969                 chip_name = "fiji";
 970                 break;
 971         case CHIP_STONEY:
 972                 chip_name = "stoney";
 973                 break;
 974         case CHIP_POLARIS10:
 975                 chip_name = "polaris10";
 976                 break;
 977         case CHIP_POLARIS11:
 978                 chip_name = "polaris11";
 979                 break;
 980         case CHIP_POLARIS12:
 981                 chip_name = "polaris12";
 982                 break;
 983         case CHIP_VEGAM:
 984                 chip_name = "vegam";
 985                 break;
 986         default:
 987                 BUG();
 988         }
 989
 990         if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
 991                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp_2.bin", chip_name);
 992                 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
 993                 if (err == -ENOENT) {
 994                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
 995                         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
 996                 }
 997         } else {
 998                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
 999                 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1000         }
1001         if (err)
1002                 goto out;
1003         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1004         if (err)
1005                 goto out;
1006         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1007         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1008         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1009
1010         if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1011                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me_2.bin", chip_name);
1012                 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1013                 if (err == -ENOENT) {
1014                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1015                         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1016                 }
1017         } else {
1018                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1019                 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1020         }
1021         if (err)
1022                 goto out;
1023         err = amdgpu_ucode_validate(adev->gfx.me_fw);
1024         if (err)
1025                 goto out;
1026         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1027         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1028
1029         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1030
1031         if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1032                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce_2.bin", chip_name);
1033                 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1034                 if (err == -ENOENT) {
1035                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1036                         err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1037                 }
1038         } else {
1039                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1040                 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1041         }
1042         if (err)
1043                 goto out;
1044         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1045         if (err)
1046                 goto out;
1047         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1048         adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1049         adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1050
1051         /*
1052          * Support for MCBP/Virtualization in combination with chained IBs is
1053          * formal released on feature version #46
1054          */
1055         if (adev->gfx.ce_feature_version >= 46 &&
1056             adev->gfx.pfp_feature_version >= 46) {
1057                 adev->virt.chained_ib_support = true;
1058                 DRM_INFO("Chained IB support enabled!\n");
1059         } else
1060                 adev->virt.chained_ib_support = false;
1061
1062         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1063         err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1064         if (err)
1065                 goto out;
1066         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1067         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1068         adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1069         adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1070
1071         adev->gfx.rlc.save_and_restore_offset =
1072                         le32_to_cpu(rlc_hdr->save_and_restore_offset);
1073         adev->gfx.rlc.clear_state_descriptor_offset =
1074                         le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1075         adev->gfx.rlc.avail_scratch_ram_locations =
1076                         le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1077         adev->gfx.rlc.reg_restore_list_size =
1078                         le32_to_cpu(rlc_hdr->reg_restore_list_size);
1079         adev->gfx.rlc.reg_list_format_start =
1080                         le32_to_cpu(rlc_hdr->reg_list_format_start);
1081         adev->gfx.rlc.reg_list_format_separate_start =
1082                         le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1083         adev->gfx.rlc.starting_offsets_start =
1084                         le32_to_cpu(rlc_hdr->starting_offsets_start);
1085         adev->gfx.rlc.reg_list_format_size_bytes =
1086                         le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1087         adev->gfx.rlc.reg_list_size_bytes =
1088                         le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1089
1090         adev->gfx.rlc.register_list_format =
1091                         kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1092                                         adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1093
1094         if (!adev->gfx.rlc.register_list_format) {
1095                 err = -ENOMEM;
1096                 goto out;
1097         }
1098
1099         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1100                         le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1101         for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1102                 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1103
1104         adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1105
1106         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1107                         le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1108         for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1109                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1110
1111         if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1112                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec_2.bin", chip_name);
1113                 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1114                 if (err == -ENOENT) {
1115                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1116                         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1117                 }
1118         } else {
1119                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1120                 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1121         }
1122         if (err)
1123                 goto out;
1124         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1125         if (err)
1126                 goto out;
1127         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1128         adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1129         adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1130
1131         if ((adev->asic_type != CHIP_STONEY) &&
1132             (adev->asic_type != CHIP_TOPAZ)) {
1133                 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1134                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2_2.bin", chip_name);
1135                         err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1136                         if (err == -ENOENT) {
1137                                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1138                                 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1139                         }
1140                 } else {
1141                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1142                         err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1143                 }
1144                 if (!err) {
1145                         err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1146                         if (err)
1147                                 goto out;
1148                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1149                                 adev->gfx.mec2_fw->data;
1150                         adev->gfx.mec2_fw_version =
1151                                 le32_to_cpu(cp_hdr->header.ucode_version);
1152                         adev->gfx.mec2_feature_version =
1153                                 le32_to_cpu(cp_hdr->ucode_feature_version);
1154                 } else {
1155                         err = 0;
1156                         adev->gfx.mec2_fw = NULL;
1157                 }
1158         }
1159
1160         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1161         info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1162         info->fw = adev->gfx.pfp_fw;
1163         header = (const struct common_firmware_header *)info->fw->data;
1164         adev->firmware.fw_size +=
1165                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1166
1167         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1168         info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1169         info->fw = adev->gfx.me_fw;
1170         header = (const struct common_firmware_header *)info->fw->data;
1171         adev->firmware.fw_size +=
1172                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1173
1174         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1175         info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1176         info->fw = adev->gfx.ce_fw;
1177         header = (const struct common_firmware_header *)info->fw->data;
1178         adev->firmware.fw_size +=
1179                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1180
1181         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1182         info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1183         info->fw = adev->gfx.rlc_fw;
1184         header = (const struct common_firmware_header *)info->fw->data;
1185         adev->firmware.fw_size +=
1186                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1187
1188         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1189         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1190         info->fw = adev->gfx.mec_fw;
1191         header = (const struct common_firmware_header *)info->fw->data;
1192         adev->firmware.fw_size +=
1193                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1194
1195         /* we need account JT in */
1196         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1197         adev->firmware.fw_size +=
1198                 ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);
1199
1200         if (amdgpu_sriov_vf(adev)) {
1201                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE];
1202                 info->ucode_id = AMDGPU_UCODE_ID_STORAGE;
1203                 info->fw = adev->gfx.mec_fw;
1204                 adev->firmware.fw_size +=
1205                         ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE);
1206         }
1207
1208         if (adev->gfx.mec2_fw) {
1209                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1210                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1211                 info->fw = adev->gfx.mec2_fw;
1212                 header = (const struct common_firmware_header *)info->fw->data;
1213                 adev->firmware.fw_size +=
1214                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1215         }
1216
1217 out:
1218         if (err) {
1219                 dev_err(adev->dev,
1220                         "gfx8: Failed to load firmware \"%s\"\n",
1221                         fw_name);
1222                 release_firmware(adev->gfx.pfp_fw);
1223                 adev->gfx.pfp_fw = NULL;
1224                 release_firmware(adev->gfx.me_fw);
1225                 adev->gfx.me_fw = NULL;
1226                 release_firmware(adev->gfx.ce_fw);
1227                 adev->gfx.ce_fw = NULL;
1228                 release_firmware(adev->gfx.rlc_fw);
1229                 adev->gfx.rlc_fw = NULL;
1230                 release_firmware(adev->gfx.mec_fw);
1231                 adev->gfx.mec_fw = NULL;
1232                 release_firmware(adev->gfx.mec2_fw);
1233                 adev->gfx.mec2_fw = NULL;
1234         }
1235         return err;
1236 }
1237
1238 static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev,
1239                                     volatile u32 *buffer)
1240 {
1241         u32 count = 0, i;
1242         const struct cs_section_def *sect = NULL;
1243         const struct cs_extent_def *ext = NULL;
1244
1245         if (adev->gfx.rlc.cs_data == NULL)
1246                 return;
1247         if (buffer == NULL)
1248                 return;
1249
1250         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1251         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1252
1253         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1254         buffer[count++] = cpu_to_le32(0x80000000);
1255         buffer[count++] = cpu_to_le32(0x80000000);
1256
1257         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1258                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1259                         if (sect->id == SECT_CONTEXT) {
1260                                 buffer[count++] =
1261                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1262                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1263                                                 PACKET3_SET_CONTEXT_REG_START);
1264                                 for (i = 0; i < ext->reg_count; i++)
1265                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1266                         } else {
1267                                 return;
1268                         }
1269                 }
1270         }
1271
1272         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
1273         buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG -
1274                         PACKET3_SET_CONTEXT_REG_START);
1275         buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);
1276         buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1);
1277
1278         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1279         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1280
1281         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1282         buffer[count++] = cpu_to_le32(0);
1283 }
1284
1285 static int gfx_v8_0_cp_jump_table_num(struct amdgpu_device *adev)
1286 {
1287         if (adev->asic_type == CHIP_CARRIZO)
1288                 return 5;
1289         else
1290                 return 4;
1291 }
1292
1293 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
1294 {
1295         const struct cs_section_def *cs_data;
1296         int r;
1297
1298         adev->gfx.rlc.cs_data = vi_cs_data;
1299
1300         cs_data = adev->gfx.rlc.cs_data;
1301
1302         if (cs_data) {
1303                 /* init clear state block */
1304                 r = amdgpu_gfx_rlc_init_csb(adev);
1305                 if (r)
1306                         return r;
1307         }
1308
1309         if ((adev->asic_type == CHIP_CARRIZO) ||
1310             (adev->asic_type == CHIP_STONEY)) {
1311                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1312                 r = amdgpu_gfx_rlc_init_cpt(adev);
1313                 if (r)
1314                         return r;
1315         }
1316
1317         /* init spm vmid with 0xf */
1318         if (adev->gfx.rlc.funcs->update_spm_vmid)
1319                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1320
1321         return 0;
1322 }
1323
1324 static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
1325 {
1326         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1327 }
1328
1329 static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
1330 {
1331         int r;
1332         u32 *hpd;
1333         size_t mec_hpd_size;
1334
1335         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1336
1337         /* take ownership of the relevant compute queues */
1338         amdgpu_gfx_compute_queue_acquire(adev);
1339
1340         mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE;
1341         if (mec_hpd_size) {
1342                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1343                                               AMDGPU_GEM_DOMAIN_VRAM |
1344                                               AMDGPU_GEM_DOMAIN_GTT,
1345                                               &adev->gfx.mec.hpd_eop_obj,
1346                                               &adev->gfx.mec.hpd_eop_gpu_addr,
1347                                               (void **)&hpd);
1348                 if (r) {
1349                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1350                         return r;
1351                 }
1352
1353                 memset(hpd, 0, mec_hpd_size);
1354
1355                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1356                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1357         }
1358
1359         return 0;
1360 }
1361
1362 static const u32 vgpr_init_compute_shader[] =
1363 {
1364         0x7e000209, 0x7e020208,
1365         0x7e040207, 0x7e060206,
1366         0x7e080205, 0x7e0a0204,
1367         0x7e0c0203, 0x7e0e0202,
1368         0x7e100201, 0x7e120200,
1369         0x7e140209, 0x7e160208,
1370         0x7e180207, 0x7e1a0206,
1371         0x7e1c0205, 0x7e1e0204,
1372         0x7e200203, 0x7e220202,
1373         0x7e240201, 0x7e260200,
1374         0x7e280209, 0x7e2a0208,
1375         0x7e2c0207, 0x7e2e0206,
1376         0x7e300205, 0x7e320204,
1377         0x7e340203, 0x7e360202,
1378         0x7e380201, 0x7e3a0200,
1379         0x7e3c0209, 0x7e3e0208,
1380         0x7e400207, 0x7e420206,
1381         0x7e440205, 0x7e460204,
1382         0x7e480203, 0x7e4a0202,
1383         0x7e4c0201, 0x7e4e0200,
1384         0x7e500209, 0x7e520208,
1385         0x7e540207, 0x7e560206,
1386         0x7e580205, 0x7e5a0204,
1387         0x7e5c0203, 0x7e5e0202,
1388         0x7e600201, 0x7e620200,
1389         0x7e640209, 0x7e660208,
1390         0x7e680207, 0x7e6a0206,
1391         0x7e6c0205, 0x7e6e0204,
1392         0x7e700203, 0x7e720202,
1393         0x7e740201, 0x7e760200,
1394         0x7e780209, 0x7e7a0208,
1395         0x7e7c0207, 0x7e7e0206,
1396         0xbf8a0000, 0xbf810000,
1397 };
1398
1399 static const u32 sgpr_init_compute_shader[] =
1400 {
1401         0xbe8a0100, 0xbe8c0102,
1402         0xbe8e0104, 0xbe900106,
1403         0xbe920108, 0xbe940100,
1404         0xbe960102, 0xbe980104,
1405         0xbe9a0106, 0xbe9c0108,
1406         0xbe9e0100, 0xbea00102,
1407         0xbea20104, 0xbea40106,
1408         0xbea60108, 0xbea80100,
1409         0xbeaa0102, 0xbeac0104,
1410         0xbeae0106, 0xbeb00108,
1411         0xbeb20100, 0xbeb40102,
1412         0xbeb60104, 0xbeb80106,
1413         0xbeba0108, 0xbebc0100,
1414         0xbebe0102, 0xbec00104,
1415         0xbec20106, 0xbec40108,
1416         0xbec60100, 0xbec80102,
1417         0xbee60004, 0xbee70005,
1418         0xbeea0006, 0xbeeb0007,
1419         0xbee80008, 0xbee90009,
1420         0xbefc0000, 0xbf8a0000,
1421         0xbf810000, 0x00000000,
1422 };
1423
1424 static const u32 vgpr_init_regs[] =
1425 {
1426         mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff,
1427         mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */
1428         mmCOMPUTE_NUM_THREAD_X, 256*4,
1429         mmCOMPUTE_NUM_THREAD_Y, 1,
1430         mmCOMPUTE_NUM_THREAD_Z, 1,
1431         mmCOMPUTE_PGM_RSRC1, 0x100004f, /* VGPRS=15 (64 logical VGPRs), SGPRS=1 (16 SGPRs), BULKY=1 */
1432         mmCOMPUTE_PGM_RSRC2, 20,
1433         mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1434         mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1435         mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1436         mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1437         mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1438         mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1439         mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1440         mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1441         mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1442         mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1443 };
1444
1445 static const u32 sgpr1_init_regs[] =
1446 {
1447         mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f,
1448         mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */
1449         mmCOMPUTE_NUM_THREAD_X, 256*5,
1450         mmCOMPUTE_NUM_THREAD_Y, 1,
1451         mmCOMPUTE_NUM_THREAD_Z, 1,
1452         mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */
1453         mmCOMPUTE_PGM_RSRC2, 20,
1454         mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1455         mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1456         mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1457         mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1458         mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1459         mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1460         mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1461         mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1462         mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1463         mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1464 };
1465
1466 static const u32 sgpr2_init_regs[] =
1467 {
1468         mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0,
1469         mmCOMPUTE_RESOURCE_LIMITS, 0x1000000,
1470         mmCOMPUTE_NUM_THREAD_X, 256*5,
1471         mmCOMPUTE_NUM_THREAD_Y, 1,
1472         mmCOMPUTE_NUM_THREAD_Z, 1,
1473         mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */
1474         mmCOMPUTE_PGM_RSRC2, 20,
1475         mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1476         mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1477         mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1478         mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1479         mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1480         mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1481         mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1482         mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1483         mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1484         mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1485 };
1486
1487 static const u32 sec_ded_counter_registers[] =
1488 {
1489         mmCPC_EDC_ATC_CNT,
1490         mmCPC_EDC_SCRATCH_CNT,
1491         mmCPC_EDC_UCODE_CNT,
1492         mmCPF_EDC_ATC_CNT,
1493         mmCPF_EDC_ROQ_CNT,
1494         mmCPF_EDC_TAG_CNT,
1495         mmCPG_EDC_ATC_CNT,
1496         mmCPG_EDC_DMA_CNT,
1497         mmCPG_EDC_TAG_CNT,
1498         mmDC_EDC_CSINVOC_CNT,
1499         mmDC_EDC_RESTORE_CNT,
1500         mmDC_EDC_STATE_CNT,
1501         mmGDS_EDC_CNT,
1502         mmGDS_EDC_GRBM_CNT,
1503         mmGDS_EDC_OA_DED,
1504         mmSPI_EDC_CNT,
1505         mmSQC_ATC_EDC_GATCL1_CNT,
1506         mmSQC_EDC_CNT,
1507         mmSQ_EDC_DED_CNT,
1508         mmSQ_EDC_INFO,
1509         mmSQ_EDC_SEC_CNT,
1510         mmTCC_EDC_CNT,
1511         mmTCP_ATC_EDC_GATCL1_CNT,
1512         mmTCP_EDC_CNT,
1513         mmTD_EDC_CNT
1514 };
1515
1516 static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
1517 {
1518         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
1519         struct amdgpu_ib ib;
1520         struct dma_fence *f = NULL;
1521         int r, i;
1522         u32 tmp;
1523         unsigned total_size, vgpr_offset, sgpr_offset;
1524         u64 gpu_addr;
1525
1526         /* only supported on CZ */
1527         if (adev->asic_type != CHIP_CARRIZO)
1528                 return 0;
1529
1530         /* bail if the compute ring is not ready */
1531         if (!ring->sched.ready)
1532                 return 0;
1533
1534         tmp = RREG32(mmGB_EDC_MODE);
1535         WREG32(mmGB_EDC_MODE, 0);
1536
1537         total_size =
1538                 (((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1539         total_size +=
1540                 (((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1541         total_size +=
1542                 (((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1543         total_size = ALIGN(total_size, 256);
1544         vgpr_offset = total_size;
1545         total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256);
1546         sgpr_offset = total_size;
1547         total_size += sizeof(sgpr_init_compute_shader);
1548
1549         /* allocate an indirect buffer to put the commands in */
1550         memset(&ib, 0, sizeof(ib));
1551         r = amdgpu_ib_get(adev, NULL, total_size,
1552                                         AMDGPU_IB_POOL_DIRECT, &ib);
1553         if (r) {
1554                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
1555                 return r;
1556         }
1557
1558         /* load the compute shaders */
1559         for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)
1560                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
1561
1562         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
1563                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
1564
1565         /* init the ib length to 0 */
1566         ib.length_dw = 0;
1567
1568         /* VGPR */
1569         /* write the register state for the compute dispatch */
1570         for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) {
1571                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1572                 ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START;
1573                 ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1];
1574         }
1575         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1576         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
1577         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1578         ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1579         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1580         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1581
1582         /* write dispatch packet */
1583         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1584         ib.ptr[ib.length_dw++] = 8; /* x */
1585         ib.ptr[ib.length_dw++] = 1; /* y */
1586         ib.ptr[ib.length_dw++] = 1; /* z */
1587         ib.ptr[ib.length_dw++] =
1588                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1589
1590         /* write CS partial flush packet */
1591         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1592         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1593
1594         /* SGPR1 */
1595         /* write the register state for the compute dispatch */
1596         for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) {
1597                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1598                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START;
1599                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1];
1600         }
1601         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1602         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1603         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1604         ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1605         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1606         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1607
1608         /* write dispatch packet */
1609         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1610         ib.ptr[ib.length_dw++] = 8; /* x */
1611         ib.ptr[ib.length_dw++] = 1; /* y */
1612         ib.ptr[ib.length_dw++] = 1; /* z */
1613         ib.ptr[ib.length_dw++] =
1614                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1615
1616         /* write CS partial flush packet */
1617         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1618         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1619
1620         /* SGPR2 */
1621         /* write the register state for the compute dispatch */
1622         for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) {
1623                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1624                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START;
1625                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1];
1626         }
1627         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1628         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1629         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1630         ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1631         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1632         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1633
1634         /* write dispatch packet */
1635         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1636         ib.ptr[ib.length_dw++] = 8; /* x */
1637         ib.ptr[ib.length_dw++] = 1; /* y */
1638         ib.ptr[ib.length_dw++] = 1; /* z */
1639         ib.ptr[ib.length_dw++] =
1640                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1641
1642         /* write CS partial flush packet */
1643         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1644         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1645
1646         /* shedule the ib on the ring */
1647         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1648         if (r) {
1649                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
1650                 goto fail;
1651         }
1652
1653         /* wait for the GPU to finish processing the IB */
1654         r = dma_fence_wait(f, false);
1655         if (r) {
1656                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
1657                 goto fail;
1658         }
1659
1660         tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2);
1661         tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1);
1662         WREG32(mmGB_EDC_MODE, tmp);
1663
1664         tmp = RREG32(mmCC_GC_EDC_CONFIG);
1665         tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1;
1666         WREG32(mmCC_GC_EDC_CONFIG, tmp);
1667
1668
1669         /* read back registers to clear the counters */
1670         for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++)
1671                 RREG32(sec_ded_counter_registers[i]);
1672
1673 fail:
1674         amdgpu_ib_free(adev, &ib, NULL);
1675         dma_fence_put(f);
1676
1677         return r;
1678 }
1679
1680 static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
1681 {
1682         u32 gb_addr_config;
1683         u32 mc_arb_ramcfg;
1684         u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;
1685         u32 tmp;
1686         int ret;
1687
1688         switch (adev->asic_type) {
1689         case CHIP_TOPAZ:
1690                 adev->gfx.config.max_shader_engines = 1;
1691                 adev->gfx.config.max_tile_pipes = 2;
1692                 adev->gfx.config.max_cu_per_sh = 6;
1693                 adev->gfx.config.max_sh_per_se = 1;
1694                 adev->gfx.config.max_backends_per_se = 2;
1695                 adev->gfx.config.max_texture_channel_caches = 2;
1696                 adev->gfx.config.max_gprs = 256;
1697                 adev->gfx.config.max_gs_threads = 32;
1698                 adev->gfx.config.max_hw_contexts = 8;
1699
1700                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1701                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1702                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1703                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1704                 gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN;
1705                 break;
1706         case CHIP_FIJI:
1707                 adev->gfx.config.max_shader_engines = 4;
1708                 adev->gfx.config.max_tile_pipes = 16;
1709                 adev->gfx.config.max_cu_per_sh = 16;
1710                 adev->gfx.config.max_sh_per_se = 1;
1711                 adev->gfx.config.max_backends_per_se = 4;
1712                 adev->gfx.config.max_texture_channel_caches = 16;
1713                 adev->gfx.config.max_gprs = 256;
1714                 adev->gfx.config.max_gs_threads = 32;
1715                 adev->gfx.config.max_hw_contexts = 8;
1716
1717                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1718                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1719                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1720                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1721                 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1722                 break;
1723         case CHIP_POLARIS11:
1724         case CHIP_POLARIS12:
1725                 ret = amdgpu_atombios_get_gfx_info(adev);
1726                 if (ret)
1727                         return ret;
1728                 adev->gfx.config.max_gprs = 256;
1729                 adev->gfx.config.max_gs_threads = 32;
1730                 adev->gfx.config.max_hw_contexts = 8;
1731
1732                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1733                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1734                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1735                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1736                 gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN;
1737                 break;
1738         case CHIP_POLARIS10:
1739         case CHIP_VEGAM:
1740                 ret = amdgpu_atombios_get_gfx_info(adev);
1741                 if (ret)
1742                         return ret;
1743                 adev->gfx.config.max_gprs = 256;
1744                 adev->gfx.config.max_gs_threads = 32;
1745                 adev->gfx.config.max_hw_contexts = 8;
1746
1747                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1748                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1749                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1750                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1751                 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1752                 break;
1753         case CHIP_TONGA:
1754                 adev->gfx.config.max_shader_engines = 4;
1755                 adev->gfx.config.max_tile_pipes = 8;
1756                 adev->gfx.config.max_cu_per_sh = 8;
1757                 adev->gfx.config.max_sh_per_se = 1;
1758                 adev->gfx.config.max_backends_per_se = 2;
1759                 adev->gfx.config.max_texture_channel_caches = 8;
1760                 adev->gfx.config.max_gprs = 256;
1761                 adev->gfx.config.max_gs_threads = 32;
1762                 adev->gfx.config.max_hw_contexts = 8;
1763
1764                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1765                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1766                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1767                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1768                 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1769                 break;
1770         case CHIP_CARRIZO:
1771                 adev->gfx.config.max_shader_engines = 1;
1772                 adev->gfx.config.max_tile_pipes = 2;
1773                 adev->gfx.config.max_sh_per_se = 1;
1774                 adev->gfx.config.max_backends_per_se = 2;
1775                 adev->gfx.config.max_cu_per_sh = 8;
1776                 adev->gfx.config.max_texture_channel_caches = 2;
1777                 adev->gfx.config.max_gprs = 256;
1778                 adev->gfx.config.max_gs_threads = 32;
1779                 adev->gfx.config.max_hw_contexts = 8;
1780
1781                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1782                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1783                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1784                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1785                 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1786                 break;
1787         case CHIP_STONEY:
1788                 adev->gfx.config.max_shader_engines = 1;
1789                 adev->gfx.config.max_tile_pipes = 2;
1790                 adev->gfx.config.max_sh_per_se = 1;
1791                 adev->gfx.config.max_backends_per_se = 1;
1792                 adev->gfx.config.max_cu_per_sh = 3;
1793                 adev->gfx.config.max_texture_channel_caches = 2;
1794                 adev->gfx.config.max_gprs = 256;
1795                 adev->gfx.config.max_gs_threads = 16;
1796                 adev->gfx.config.max_hw_contexts = 8;
1797
1798                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1799                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1800                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1801                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1802                 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1803                 break;
1804         default:
1805                 adev->gfx.config.max_shader_engines = 2;
1806                 adev->gfx.config.max_tile_pipes = 4;
1807                 adev->gfx.config.max_cu_per_sh = 2;
1808                 adev->gfx.config.max_sh_per_se = 1;
1809                 adev->gfx.config.max_backends_per_se = 2;
1810                 adev->gfx.config.max_texture_channel_caches = 4;
1811                 adev->gfx.config.max_gprs = 256;
1812                 adev->gfx.config.max_gs_threads = 32;
1813                 adev->gfx.config.max_hw_contexts = 8;
1814
1815                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1816                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1817                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1818                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1819                 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1820                 break;
1821         }
1822
1823         adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
1824         mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
1825
1826         adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg,
1827                                 MC_ARB_RAMCFG, NOOFBANK);
1828         adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg,
1829                                 MC_ARB_RAMCFG, NOOFRANKS);
1830
1831         adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
1832         adev->gfx.config.mem_max_burst_length_bytes = 256;
1833         if (adev->flags & AMD_IS_APU) {
1834                 /* Get memory bank mapping mode. */
1835                 tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
1836                 dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
1837                 dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
1838
1839                 tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);
1840                 dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
1841                 dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
1842
1843                 /* Validate settings in case only one DIMM installed. */
1844                 if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))
1845                         dimm00_addr_map = 0;
1846                 if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))
1847                         dimm01_addr_map = 0;
1848                 if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))
1849                         dimm10_addr_map = 0;
1850                 if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))
1851                         dimm11_addr_map = 0;
1852
1853                 /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */
1854                 /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */
1855                 if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))
1856                         adev->gfx.config.mem_row_size_in_kb = 2;
1857                 else
1858                         adev->gfx.config.mem_row_size_in_kb = 1;
1859         } else {
1860                 tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS);
1861                 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1862                 if (adev->gfx.config.mem_row_size_in_kb > 4)
1863                         adev->gfx.config.mem_row_size_in_kb = 4;
1864         }
1865
1866         adev->gfx.config.shader_engine_tile_size = 32;
1867         adev->gfx.config.num_gpus = 1;
1868         adev->gfx.config.multi_gpu_tile_size = 64;
1869
1870         /* fix up row size */
1871         switch (adev->gfx.config.mem_row_size_in_kb) {
1872         case 1:
1873         default:
1874                 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0);
1875                 break;
1876         case 2:
1877                 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1);
1878                 break;
1879         case 4:
1880                 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2);
1881                 break;
1882         }
1883         adev->gfx.config.gb_addr_config = gb_addr_config;
1884
1885         return 0;
1886 }
1887
1888 static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1889                                         int mec, int pipe, int queue)
1890 {
1891         int r;
1892         unsigned irq_type;
1893         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
1894         unsigned int hw_prio;
1895
1896         ring = &adev->gfx.compute_ring[ring_id];
1897
1898         /* mec0 is me1 */
1899         ring->me = mec + 1;
1900         ring->pipe = pipe;
1901         ring->queue = queue;
1902
1903         ring->ring_obj = NULL;
1904         ring->use_doorbell = true;
1905         ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id;
1906         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1907                                 + (ring_id * GFX8_MEC_HPD_SIZE);
1908         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1909
1910         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1911                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1912                 + ring->pipe;
1913
1914         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1915                         AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
1916         /* type-2 packets are deprecated on MEC, use type-3 instead */
1917         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1918                              hw_prio, NULL);
1919         if (r)
1920                 return r;
1921
1922
1923         return 0;
1924 }
1925
1926 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work);
1927
1928 static int gfx_v8_0_sw_init(void *handle)
1929 {
1930         int i, j, k, r, ring_id;
1931         struct amdgpu_ring *ring;
1932         struct amdgpu_kiq *kiq;
1933         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1934
1935         switch (adev->asic_type) {
1936         case CHIP_TONGA:
1937         case CHIP_CARRIZO:
1938         case CHIP_FIJI:
1939         case CHIP_POLARIS10:
1940         case CHIP_POLARIS11:
1941         case CHIP_POLARIS12:
1942         case CHIP_VEGAM:
1943                 adev->gfx.mec.num_mec = 2;
1944                 break;
1945         case CHIP_TOPAZ:
1946         case CHIP_STONEY:
1947         default:
1948                 adev->gfx.mec.num_mec = 1;
1949                 break;
1950         }
1951
1952         adev->gfx.mec.num_pipe_per_mec = 4;
1953         adev->gfx.mec.num_queue_per_pipe = 8;
1954
1955         /* EOP Event */
1956         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq);
1957         if (r)
1958                 return r;
1959
1960         /* Privileged reg */
1961         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT,
1962                               &adev->gfx.priv_reg_irq);
1963         if (r)
1964                 return r;
1965
1966         /* Privileged inst */
1967         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT,
1968                               &adev->gfx.priv_inst_irq);
1969         if (r)
1970                 return r;
1971
1972         /* Add CP EDC/ECC irq  */
1973         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR,
1974                               &adev->gfx.cp_ecc_error_irq);
1975         if (r)
1976                 return r;
1977
1978         /* SQ interrupts. */
1979         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG,
1980                               &adev->gfx.sq_irq);
1981         if (r) {
1982                 DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r);
1983                 return r;
1984         }
1985
1986         INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func);
1987
1988         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1989
1990         r = gfx_v8_0_init_microcode(adev);
1991         if (r) {
1992                 DRM_ERROR("Failed to load gfx firmware!\n");
1993                 return r;
1994         }
1995
1996         r = adev->gfx.rlc.funcs->init(adev);
1997         if (r) {
1998                 DRM_ERROR("Failed to init rlc BOs!\n");
1999                 return r;
2000         }
2001
2002         r = gfx_v8_0_mec_init(adev);
2003         if (r) {
2004                 DRM_ERROR("Failed to init MEC BOs!\n");
2005                 return r;
2006         }
2007
2008         /* set up the gfx ring */
2009         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2010                 ring = &adev->gfx.gfx_ring[i];
2011                 ring->ring_obj = NULL;
2012                 sprintf(ring->name, "gfx");
2013                 /* no gfx doorbells on iceland */
2014                 if (adev->asic_type != CHIP_TOPAZ) {
2015                         ring->use_doorbell = true;
2016                         ring->doorbell_index = adev->doorbell_index.gfx_ring0;
2017                 }
2018
2019                 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2020                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2021                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
2022                 if (r)
2023                         return r;
2024         }
2025
2026
2027         /* set up the compute queues - allocate horizontally across pipes */
2028         ring_id = 0;
2029         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2030                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2031                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2032                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2033                                         continue;
2034
2035                                 r = gfx_v8_0_compute_ring_init(adev,
2036                                                                 ring_id,
2037                                                                 i, k, j);
2038                                 if (r)
2039                                         return r;
2040
2041                                 ring_id++;
2042                         }
2043                 }
2044         }
2045
2046         r = amdgpu_gfx_kiq_init(adev, GFX8_MEC_HPD_SIZE);
2047         if (r) {
2048                 DRM_ERROR("Failed to init KIQ BOs!\n");
2049                 return r;
2050         }
2051
2052         kiq = &adev->gfx.kiq;
2053         r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2054         if (r)
2055                 return r;
2056
2057         /* create MQD for all compute queues as well as KIQ for SRIOV case */
2058         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation));
2059         if (r)
2060                 return r;
2061
2062         adev->gfx.ce_ram_size = 0x8000;
2063
2064         r = gfx_v8_0_gpu_early_init(adev);
2065         if (r)
2066                 return r;
2067
2068         return 0;
2069 }
2070
2071 static int gfx_v8_0_sw_fini(void *handle)
2072 {
2073         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2074         int i;
2075
2076         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2077                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2078         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2079                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2080
2081         amdgpu_gfx_mqd_sw_fini(adev);
2082         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2083         amdgpu_gfx_kiq_fini(adev);
2084
2085         gfx_v8_0_mec_fini(adev);
2086         amdgpu_gfx_rlc_fini(adev);
2087         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2088                                 &adev->gfx.rlc.clear_state_gpu_addr,
2089                                 (void **)&adev->gfx.rlc.cs_ptr);
2090         if ((adev->asic_type == CHIP_CARRIZO) ||
2091             (adev->asic_type == CHIP_STONEY)) {
2092                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2093                                 &adev->gfx.rlc.cp_table_gpu_addr,
2094                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2095         }
2096         gfx_v8_0_free_microcode(adev);
2097
2098         return 0;
2099 }
2100
2101 static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
2102 {
2103         uint32_t *modearray, *mod2array;
2104         const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array);
2105         const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
2106         u32 reg_offset;
2107
2108         modearray = adev->gfx.config.tile_mode_array;
2109         mod2array = adev->gfx.config.macrotile_mode_array;
2110
2111         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2112                 modearray[reg_offset] = 0;
2113
2114         for (reg_offset = 0; reg_offset <  num_secondary_tile_mode_states; reg_offset++)
2115                 mod2array[reg_offset] = 0;
2116
2117         switch (adev->asic_type) {
2118         case CHIP_TOPAZ:
2119                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2120                                 PIPE_CONFIG(ADDR_SURF_P2) |
2121                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2122                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2123                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2124                                 PIPE_CONFIG(ADDR_SURF_P2) |
2125                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2126                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2127                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2128                                 PIPE_CONFIG(ADDR_SURF_P2) |
2129                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2130                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2131                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2132                                 PIPE_CONFIG(ADDR_SURF_P2) |
2133                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2134                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2135                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2136                                 PIPE_CONFIG(ADDR_SURF_P2) |
2137                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2138                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2139                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2140                                 PIPE_CONFIG(ADDR_SURF_P2) |
2141                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2142                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2143                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2144                                 PIPE_CONFIG(ADDR_SURF_P2) |
2145                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2146                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2147                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2148                                 PIPE_CONFIG(ADDR_SURF_P2));
2149                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2150                                 PIPE_CONFIG(ADDR_SURF_P2) |
2151                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2152                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2153                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2154                                  PIPE_CONFIG(ADDR_SURF_P2) |
2155                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2156                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2157                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2158                                  PIPE_CONFIG(ADDR_SURF_P2) |
2159                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2160                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2161                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2162                                  PIPE_CONFIG(ADDR_SURF_P2) |
2163                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2164                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2165                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2166                                  PIPE_CONFIG(ADDR_SURF_P2) |
2167                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2168                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2169                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2170                                  PIPE_CONFIG(ADDR_SURF_P2) |
2171                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2172                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2173                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2174                                  PIPE_CONFIG(ADDR_SURF_P2) |
2175                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2176                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2177                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2178                                  PIPE_CONFIG(ADDR_SURF_P2) |
2179                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2180                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2181                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2182                                  PIPE_CONFIG(ADDR_SURF_P2) |
2183                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2184                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2185                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2186                                  PIPE_CONFIG(ADDR_SURF_P2) |
2187                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2188                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2189                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2190                                  PIPE_CONFIG(ADDR_SURF_P2) |
2191                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2192                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2193                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2194                                  PIPE_CONFIG(ADDR_SURF_P2) |
2195                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2196                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2197                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2198                                  PIPE_CONFIG(ADDR_SURF_P2) |
2199                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2200                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2201                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2202                                  PIPE_CONFIG(ADDR_SURF_P2) |
2203                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2204                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2205                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2206                                  PIPE_CONFIG(ADDR_SURF_P2) |
2207                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2208                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2209                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2210                                  PIPE_CONFIG(ADDR_SURF_P2) |
2211                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2212                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2213                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2214                                  PIPE_CONFIG(ADDR_SURF_P2) |
2215                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2216                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2217                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2218                                  PIPE_CONFIG(ADDR_SURF_P2) |
2219                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2220                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2221
2222                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2223                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2224                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2225                                 NUM_BANKS(ADDR_SURF_8_BANK));
2226                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2227                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2228                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2229                                 NUM_BANKS(ADDR_SURF_8_BANK));
2230                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2231                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2232                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2233                                 NUM_BANKS(ADDR_SURF_8_BANK));
2234                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2235                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2236                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2237                                 NUM_BANKS(ADDR_SURF_8_BANK));
2238                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2239                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2240                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2241                                 NUM_BANKS(ADDR_SURF_8_BANK));
2242                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2243                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2244                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2245                                 NUM_BANKS(ADDR_SURF_8_BANK));
2246                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2247                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2248                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2249                                 NUM_BANKS(ADDR_SURF_8_BANK));
2250                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2251                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2252                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2253                                 NUM_BANKS(ADDR_SURF_16_BANK));
2254                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2255                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2256                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2257                                 NUM_BANKS(ADDR_SURF_16_BANK));
2258                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2259                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2260                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2261                                  NUM_BANKS(ADDR_SURF_16_BANK));
2262                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2263                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2264                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2265                                  NUM_BANKS(ADDR_SURF_16_BANK));
2266                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2267                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2268                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2269                                  NUM_BANKS(ADDR_SURF_16_BANK));
2270                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2271                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2272                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2273                                  NUM_BANKS(ADDR_SURF_16_BANK));
2274                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2275                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2276                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2277                                  NUM_BANKS(ADDR_SURF_8_BANK));
2278
2279                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2280                         if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
2281                             reg_offset != 23)
2282                                 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2283
2284                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2285                         if (reg_offset != 7)
2286                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2287
2288                 break;
2289         case CHIP_FIJI:
2290         case CHIP_VEGAM:
2291                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2292                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2293                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2294                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2295                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2296                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2297                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2298                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2299                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2300                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2301                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2302                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2303                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2304                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2305                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2306                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2307                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2308                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2309                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2310                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2311                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2312                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2313                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2314                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2315                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2316                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2317                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2318                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2319                 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2320                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2321                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2322                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2323                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2324                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2325                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2326                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2327                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2328                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2329                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2330                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2331                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2332                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2333                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2334                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2335                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2336                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2337                 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2338                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2339                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2340                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2341                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2342                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2343                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2344                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2345                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2346                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2347                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2348                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2349                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2350                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2351                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2352                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2353                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2354                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2355                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2356                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2357                 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2358                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2359                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2360                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2361                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2362                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2363                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2364                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2365                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2366                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2367                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2368                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2369                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2370                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2371                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2372                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2373                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2374                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2375                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2376                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2377                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2378                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2379                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2380                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2381                 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2382                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2383                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2384                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2385                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2386                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2387                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2388                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2389                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2390                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2391                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2392                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2393                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2394                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2395                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2396                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2397                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2398                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2399                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2400                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2401                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2403                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2404                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2405                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2406                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2408                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2409                 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2410                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2411                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2412                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2413
2414                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2415                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2416                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2417                                 NUM_BANKS(ADDR_SURF_8_BANK));
2418                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2419                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2420                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2421                                 NUM_BANKS(ADDR_SURF_8_BANK));
2422                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2423                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2424                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2425                                 NUM_BANKS(ADDR_SURF_8_BANK));
2426                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2427                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2428                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2429                                 NUM_BANKS(ADDR_SURF_8_BANK));
2430                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2431                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2432                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2433                                 NUM_BANKS(ADDR_SURF_8_BANK));
2434                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2436                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2437                                 NUM_BANKS(ADDR_SURF_8_BANK));
2438                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2439                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2440                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2441                                 NUM_BANKS(ADDR_SURF_8_BANK));
2442                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2443                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2444                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2445                                 NUM_BANKS(ADDR_SURF_8_BANK));
2446                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2447                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2448                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2449                                 NUM_BANKS(ADDR_SURF_8_BANK));
2450                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2451                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2452                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2453                                  NUM_BANKS(ADDR_SURF_8_BANK));
2454                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2455                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2456                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2457                                  NUM_BANKS(ADDR_SURF_8_BANK));
2458                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2459                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2460                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2461                                  NUM_BANKS(ADDR_SURF_8_BANK));
2462                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2463                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2464                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2465                                  NUM_BANKS(ADDR_SURF_8_BANK));
2466                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2467                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2468                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2469                                  NUM_BANKS(ADDR_SURF_4_BANK));
2470
2471                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2472                         WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2473
2474                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2475                         if (reg_offset != 7)
2476                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2477
2478                 break;
2479         case CHIP_TONGA:
2480                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2481                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2482                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2483                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2484                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2485                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2486                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2487                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2488                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2489                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2490                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2491                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2492                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2493                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2494                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2495                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2496                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2497                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2498                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2499                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2500                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2501                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2502                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2503                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2504                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2505                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2506                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2507                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2508                 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2509                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2510                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2511                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2512                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2513                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2514                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2515                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2516                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2517                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2518                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2519                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2520                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2521                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2522                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2523                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2524                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2525                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2526                 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2527                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2529                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2530                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2531                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2532                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2533                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2534                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2535                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2536                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2537                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2538                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2539                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2541                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2542                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2544                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2545                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2546                 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2547                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2548                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2549                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2550                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2551                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2553                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2554                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2555                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2556                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2557                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2558                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2559                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2560                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2561                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2562                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2563                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2564                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2565                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2566                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2567                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2568                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2569                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2570                 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2571                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2572                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2573                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2574                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2575                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2576                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2577                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2578                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2579                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2580                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2581                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2582                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2583                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2584                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2585                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2586                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2587                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2588                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2589                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2590                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2591                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2592                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2593                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2594                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2595                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2597                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2598                 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2599                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2600                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2601                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2602
2603                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2604                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2605                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2606                                 NUM_BANKS(ADDR_SURF_16_BANK));
2607                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2609                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2610                                 NUM_BANKS(ADDR_SURF_16_BANK));
2611                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2613                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2614                                 NUM_BANKS(ADDR_SURF_16_BANK));
2615                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2617                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2618                                 NUM_BANKS(ADDR_SURF_16_BANK));
2619                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2621                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2622                                 NUM_BANKS(ADDR_SURF_16_BANK));
2623                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2624                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2625                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2626                                 NUM_BANKS(ADDR_SURF_16_BANK));
2627                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2630                                 NUM_BANKS(ADDR_SURF_16_BANK));
2631                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2633                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2634                                 NUM_BANKS(ADDR_SURF_16_BANK));
2635                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2636                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2637                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2638                                 NUM_BANKS(ADDR_SURF_16_BANK));
2639                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2640                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2641                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2642                                  NUM_BANKS(ADDR_SURF_16_BANK));
2643                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2644                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2645                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2646                                  NUM_BANKS(ADDR_SURF_16_BANK));
2647                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2649                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2650                                  NUM_BANKS(ADDR_SURF_8_BANK));
2651                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2652                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2653                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2654                                  NUM_BANKS(ADDR_SURF_4_BANK));
2655                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2656                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2657                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2658                                  NUM_BANKS(ADDR_SURF_4_BANK));
2659
2660                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2661                         WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2662
2663                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2664                         if (reg_offset != 7)
2665                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2666
2667                 break;
2668         case CHIP_POLARIS11:
2669         case CHIP_POLARIS12:
2670                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2671                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2672                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2673                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2674                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2675                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2676                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2677                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2678                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2679                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2680                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2681                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2682                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2683                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2684                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2685                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2686                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2687                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2688                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2689                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2690                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2691                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2692                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2693                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2694                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2695                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2696                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2697                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2698                 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2699                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2700                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2701                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2702                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2703                                 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2704                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2705                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2706                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2707                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2708                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2709                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2710                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2711                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2712                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2713                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2714                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2715                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2716                 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2717                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2718                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2719                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2720                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2721                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2722                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2723                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2724                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2725                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2726                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2727                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2728                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2729                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2730                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2731                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2732                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2733                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2734                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2735                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2736                 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2737                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2738                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2739                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2740                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2741                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2742                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2743                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2744                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2745                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2746                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2747                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2748                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2749                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2750                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2751                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2752                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2753                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2754                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2755                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2756                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2757                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2758                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2759                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2760                 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2761                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2762                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2763                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2764                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2765                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2766                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2767                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2768                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2769                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2771                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2772                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2773                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2774                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2775                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2776                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2777                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2778                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2779                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2780                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2781                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2782                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2783                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2784                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2785                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2786                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2787                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2788                 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2789                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2790                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2791                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2792
2793                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2794                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2795                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2796                                 NUM_BANKS(ADDR_SURF_16_BANK));
2797
2798                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2799                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2800                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2801                                 NUM_BANKS(ADDR_SURF_16_BANK));
2802
2803                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2804                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2805                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2806                                 NUM_BANKS(ADDR_SURF_16_BANK));
2807
2808                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2809                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2810                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2811                                 NUM_BANKS(ADDR_SURF_16_BANK));
2812
2813                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2814                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2815                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2816                                 NUM_BANKS(ADDR_SURF_16_BANK));
2817
2818                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2819                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2820                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2821                                 NUM_BANKS(ADDR_SURF_16_BANK));
2822
2823                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2824                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2825                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2826                                 NUM_BANKS(ADDR_SURF_16_BANK));
2827
2828                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2829                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2830                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2831                                 NUM_BANKS(ADDR_SURF_16_BANK));
2832
2833                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2834                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2835                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2836                                 NUM_BANKS(ADDR_SURF_16_BANK));
2837
2838                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2839                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2840                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2841                                 NUM_BANKS(ADDR_SURF_16_BANK));
2842
2843                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2844                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2845                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2846                                 NUM_BANKS(ADDR_SURF_16_BANK));
2847
2848                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2849                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2850                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2851                                 NUM_BANKS(ADDR_SURF_16_BANK));
2852
2853                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2854                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2855                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2856                                 NUM_BANKS(ADDR_SURF_8_BANK));
2857
2858                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2859                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2860                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2861                                 NUM_BANKS(ADDR_SURF_4_BANK));
2862
2863                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2864                         WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2865
2866                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2867                         if (reg_offset != 7)
2868                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2869
2870                 break;
2871         case CHIP_POLARIS10:
2872                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2873                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2874                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2875                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2876                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2877                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2878                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2879                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2880                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2881                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2882                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2883                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2884                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2885                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2886                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2887                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2888                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2889                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2890                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2891                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2892                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2893                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2894                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2895                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2896                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2897                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2898                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2899                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2900                 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2901                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2902                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2903                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2904                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2905                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2906                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2907                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2908                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2909                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2910                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2911                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2912                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2913                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2914                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2915                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2916                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2917                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2918                 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2919                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2920                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2921                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2922                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2923                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2924                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2925                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2926                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2927                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2928                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2929                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2930                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2931                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2932                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2933                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2934                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2935                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2936                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2937                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2938                 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2939                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2940                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2941                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2942                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2943                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2944                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2945                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2946                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2947                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2948                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2949                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2950                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2951                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2952                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2953                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2954                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2955                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2956                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2957                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2958                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2959                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2960                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2961                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2962                 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2963                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2964                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2965                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2966                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2967                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2968                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2969                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2970                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2971                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2972                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2973                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2974                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2975                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2976                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2977                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2978                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2979                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2980                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2981                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2982                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2983                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2984                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2985                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2986                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2987                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2988                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2989                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2990                 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2991                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2992                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2993                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2994
2995                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2996                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2997                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2998                                 NUM_BANKS(ADDR_SURF_16_BANK));
2999
3000                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3001                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3002                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3003                                 NUM_BANKS(ADDR_SURF_16_BANK));
3004
3005                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3006                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3007                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3008                                 NUM_BANKS(ADDR_SURF_16_BANK));
3009
3010                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3011                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3012                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3013                                 NUM_BANKS(ADDR_SURF_16_BANK));
3014
3015                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3016                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3017                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3018                                 NUM_BANKS(ADDR_SURF_16_BANK));
3019
3020                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3021                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3022                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3023                                 NUM_BANKS(ADDR_SURF_16_BANK));
3024
3025                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3026                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3027                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3028                                 NUM_BANKS(ADDR_SURF_16_BANK));
3029
3030                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3031                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3032                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3033                                 NUM_BANKS(ADDR_SURF_16_BANK));
3034
3035                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3036                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3037                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3038                                 NUM_BANKS(ADDR_SURF_16_BANK));
3039
3040                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3041                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3042                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3043                                 NUM_BANKS(ADDR_SURF_16_BANK));
3044
3045                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3046                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3047                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3048                                 NUM_BANKS(ADDR_SURF_16_BANK));
3049
3050                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3051                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3052                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3053                                 NUM_BANKS(ADDR_SURF_8_BANK));
3054
3055                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3056                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3057                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3058                                 NUM_BANKS(ADDR_SURF_4_BANK));
3059
3060                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3061                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3062                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3063                                 NUM_BANKS(ADDR_SURF_4_BANK));
3064
3065                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3066                         WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3067
3068                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3069                         if (reg_offset != 7)
3070                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3071
3072                 break;
3073         case CHIP_STONEY:
3074                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3075                                 PIPE_CONFIG(ADDR_SURF_P2) |
3076                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3077                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3078                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3079                                 PIPE_CONFIG(ADDR_SURF_P2) |
3080                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3081                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3082                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3083                                 PIPE_CONFIG(ADDR_SURF_P2) |
3084                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3085                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3086                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3087                                 PIPE_CONFIG(ADDR_SURF_P2) |
3088                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3089                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3090                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3091                                 PIPE_CONFIG(ADDR_SURF_P2) |
3092                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3093                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3094                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3095                                 PIPE_CONFIG(ADDR_SURF_P2) |
3096                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3097                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3098                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3099                                 PIPE_CONFIG(ADDR_SURF_P2) |
3100                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3101                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3102                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3103                                 PIPE_CONFIG(ADDR_SURF_P2));
3104                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3105                                 PIPE_CONFIG(ADDR_SURF_P2) |
3106                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3107                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3108                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3109                                  PIPE_CONFIG(ADDR_SURF_P2) |
3110                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3111                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3112                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3113                                  PIPE_CONFIG(ADDR_SURF_P2) |
3114                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3115                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3116                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3117                                  PIPE_CONFIG(ADDR_SURF_P2) |
3118                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3119                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3120                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3121                                  PIPE_CONFIG(ADDR_SURF_P2) |
3122                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3123                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3124                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3125                                  PIPE_CONFIG(ADDR_SURF_P2) |
3126                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3127                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3128                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3129                                  PIPE_CONFIG(ADDR_SURF_P2) |
3130                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3131                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3132                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3133                                  PIPE_CONFIG(ADDR_SURF_P2) |
3134                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3135                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3136                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3137                                  PIPE_CONFIG(ADDR_SURF_P2) |
3138                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3139                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3140                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3141                                  PIPE_CONFIG(ADDR_SURF_P2) |
3142                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3143                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3144                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3145                                  PIPE_CONFIG(ADDR_SURF_P2) |
3146                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3147                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3148                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3149                                  PIPE_CONFIG(ADDR_SURF_P2) |
3150                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3151                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3152                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3153                                  PIPE_CONFIG(ADDR_SURF_P2) |
3154                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3155                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3156                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3157                                  PIPE_CONFIG(ADDR_SURF_P2) |
3158                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3159                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3160                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3161                                  PIPE_CONFIG(ADDR_SURF_P2) |
3162                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3163                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3164                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3165                                  PIPE_CONFIG(ADDR_SURF_P2) |
3166                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3167                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3168                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3169                                  PIPE_CONFIG(ADDR_SURF_P2) |
3170                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3171                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3172                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3173                                  PIPE_CONFIG(ADDR_SURF_P2) |
3174                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3175                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3176
3177                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3178                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3179                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3180                                 NUM_BANKS(ADDR_SURF_8_BANK));
3181                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3182                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3183                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3184                                 NUM_BANKS(ADDR_SURF_8_BANK));
3185                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3186                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3187                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3188                                 NUM_BANKS(ADDR_SURF_8_BANK));
3189                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3190                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3191                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3192                                 NUM_BANKS(ADDR_SURF_8_BANK));
3193                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3194                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3195                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3196                                 NUM_BANKS(ADDR_SURF_8_BANK));
3197                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3198                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3199                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3200                                 NUM_BANKS(ADDR_SURF_8_BANK));
3201                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3202                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3203                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3204                                 NUM_BANKS(ADDR_SURF_8_BANK));
3205                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3206                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3207                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3208                                 NUM_BANKS(ADDR_SURF_16_BANK));
3209                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3210                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3211                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3212                                 NUM_BANKS(ADDR_SURF_16_BANK));
3213                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3214                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3215                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3216                                  NUM_BANKS(ADDR_SURF_16_BANK));
3217                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3218                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3219                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3220                                  NUM_BANKS(ADDR_SURF_16_BANK));
3221                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3222                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3223                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3224                                  NUM_BANKS(ADDR_SURF_16_BANK));
3225                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3226                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3227                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3228                                  NUM_BANKS(ADDR_SURF_16_BANK));
3229                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3230                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3231                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3232                                  NUM_BANKS(ADDR_SURF_8_BANK));
3233
3234                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3235                         if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3236                             reg_offset != 23)
3237                                 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3238
3239                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3240                         if (reg_offset != 7)
3241                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3242
3243                 break;
3244         default:
3245                 dev_warn(adev->dev,
3246                          "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n",
3247                          adev->asic_type);
3248                 fallthrough;
3249
3250         case CHIP_CARRIZO:
3251                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3252                                 PIPE_CONFIG(ADDR_SURF_P2) |
3253                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3254                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3255                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3256                                 PIPE_CONFIG(ADDR_SURF_P2) |
3257                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3258                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3259                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3260                                 PIPE_CONFIG(ADDR_SURF_P2) |
3261                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3262                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3263                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3264                                 PIPE_CONFIG(ADDR_SURF_P2) |
3265                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3266                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3267                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3268                                 PIPE_CONFIG(ADDR_SURF_P2) |
3269                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3270                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3271                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3272                                 PIPE_CONFIG(ADDR_SURF_P2) |
3273                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3274                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3275                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3276                                 PIPE_CONFIG(ADDR_SURF_P2) |
3277                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3278                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3279                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3280                                 PIPE_CONFIG(ADDR_SURF_P2));
3281                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3282                                 PIPE_CONFIG(ADDR_SURF_P2) |
3283                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3284                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3285                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3286                                  PIPE_CONFIG(ADDR_SURF_P2) |
3287                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3288                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3289                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3290                                  PIPE_CONFIG(ADDR_SURF_P2) |
3291                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3292                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3293                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3294                                  PIPE_CONFIG(ADDR_SURF_P2) |
3295                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3296                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3297                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3298                                  PIPE_CONFIG(ADDR_SURF_P2) |
3299                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3300                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3301                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3302                                  PIPE_CONFIG(ADDR_SURF_P2) |
3303                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3304                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3305                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3306                                  PIPE_CONFIG(ADDR_SURF_P2) |
3307                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3308                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3309                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3310                                  PIPE_CONFIG(ADDR_SURF_P2) |
3311                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3312                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3313                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3314                                  PIPE_CONFIG(ADDR_SURF_P2) |
3315                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3316                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3317                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3318                                  PIPE_CONFIG(ADDR_SURF_P2) |
3319                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3320                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3321                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3322                                  PIPE_CONFIG(ADDR_SURF_P2) |
3323                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3324                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3325                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3326                                  PIPE_CONFIG(ADDR_SURF_P2) |
3327                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3328                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3329                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3330                                  PIPE_CONFIG(ADDR_SURF_P2) |
3331                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3332                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3333                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3334                                  PIPE_CONFIG(ADDR_SURF_P2) |
3335                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3336                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3337                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3338                                  PIPE_CONFIG(ADDR_SURF_P2) |
3339                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3340                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3341                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3342                                  PIPE_CONFIG(ADDR_SURF_P2) |
3343                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3344                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3345                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3346                                  PIPE_CONFIG(ADDR_SURF_P2) |
3347                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3348                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3349                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3350                                  PIPE_CONFIG(ADDR_SURF_P2) |
3351                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3352                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3353
3354                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3355                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3356                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3357                                 NUM_BANKS(ADDR_SURF_8_BANK));
3358                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3359                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3360                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3361                                 NUM_BANKS(ADDR_SURF_8_BANK));
3362                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3363                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3364                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3365                                 NUM_BANKS(ADDR_SURF_8_BANK));
3366                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3367                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3368                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3369                                 NUM_BANKS(ADDR_SURF_8_BANK));
3370                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3371                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3372                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3373                                 NUM_BANKS(ADDR_SURF_8_BANK));
3374                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3375                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3376                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3377                                 NUM_BANKS(ADDR_SURF_8_BANK));
3378                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3379                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3380                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3381                                 NUM_BANKS(ADDR_SURF_8_BANK));
3382                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3383                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3384                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3385                                 NUM_BANKS(ADDR_SURF_16_BANK));
3386                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3387                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3388                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3389                                 NUM_BANKS(ADDR_SURF_16_BANK));
3390                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3391                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3392                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3393                                  NUM_BANKS(ADDR_SURF_16_BANK));
3394                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3395                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3396                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3397                                  NUM_BANKS(ADDR_SURF_16_BANK));
3398                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3399                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3400                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3401                                  NUM_BANKS(ADDR_SURF_16_BANK));
3402                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3403                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3404                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3405                                  NUM_BANKS(ADDR_SURF_16_BANK));
3406                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3407                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3408                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3409                                  NUM_BANKS(ADDR_SURF_8_BANK));
3410
3411                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3412                         if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3413                             reg_offset != 23)
3414                                 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3415
3416                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3417                         if (reg_offset != 7)
3418                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3419
3420                 break;
3421         }
3422 }
3423
3424 static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,
3425                                   u32 se_num, u32 sh_num, u32 instance)
3426 {
3427         u32 data;
3428
3429         if (instance == 0xffffffff)
3430                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
3431         else
3432                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
3433
3434         if (se_num == 0xffffffff)
3435                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
3436         else
3437                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
3438
3439         if (sh_num == 0xffffffff)
3440                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
3441         else
3442                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
3443
3444         WREG32(mmGRBM_GFX_INDEX, data);
3445 }
3446
3447 static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev,
3448                                   u32 me, u32 pipe, u32 q, u32 vm)
3449 {
3450         vi_srbm_select(adev, me, pipe, q, vm);
3451 }
3452
3453 static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)
3454 {
3455         u32 data, mask;
3456
3457         data =  RREG32(mmCC_RB_BACKEND_DISABLE) |
3458                 RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3459
3460         data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE);
3461
3462         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
3463                                          adev->gfx.config.max_sh_per_se);
3464
3465         return (~data) & mask;
3466 }
3467
3468 static void
3469 gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
3470 {
3471         switch (adev->asic_type) {
3472         case CHIP_FIJI:
3473         case CHIP_VEGAM:
3474                 *rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
3475                           RB_XSEL2(1) | PKR_MAP(2) |
3476                           PKR_XSEL(1) | PKR_YSEL(1) |
3477                           SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3);
3478                 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
3479                            SE_PAIR_YSEL(2);
3480                 break;
3481         case CHIP_TONGA:
3482         case CHIP_POLARIS10:
3483                 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3484                           SE_XSEL(1) | SE_YSEL(1);
3485                 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) |
3486                            SE_PAIR_YSEL(2);
3487                 break;
3488         case CHIP_TOPAZ:
3489         case CHIP_CARRIZO:
3490                 *rconf |= RB_MAP_PKR0(2);
3491                 *rconf1 |= 0x0;
3492                 break;
3493         case CHIP_POLARIS11:
3494         case CHIP_POLARIS12:
3495                 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3496                           SE_XSEL(1) | SE_YSEL(1);
3497                 *rconf1 |= 0x0;
3498                 break;
3499         case CHIP_STONEY:
3500                 *rconf |= 0x0;
3501                 *rconf1 |= 0x0;
3502                 break;
3503         default:
3504                 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
3505                 break;
3506         }
3507 }
3508
3509 static void
3510 gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev,
3511                                         u32 raster_config, u32 raster_config_1,
3512                                         unsigned rb_mask, unsigned num_rb)
3513 {
3514         unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
3515         unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
3516         unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
3517         unsigned rb_per_se = num_rb / num_se;
3518         unsigned se_mask[4];
3519         unsigned se;
3520
3521         se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
3522         se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
3523         se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
3524         se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
3525
3526         WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
3527         WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
3528         WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
3529
3530         if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
3531                              (!se_mask[2] && !se_mask[3]))) {
3532                 raster_config_1 &= ~SE_PAIR_MAP_MASK;
3533
3534                 if (!se_mask[0] && !se_mask[1]) {
3535                         raster_config_1 |=
3536                                 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
3537                 } else {
3538                         raster_config_1 |=
3539                                 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
3540                 }
3541         }
3542
3543         for (se = 0; se < num_se; se++) {
3544                 unsigned raster_config_se = raster_config;
3545                 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
3546                 unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
3547                 int idx = (se / 2) * 2;
3548
3549                 if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
3550                         raster_config_se &= ~SE_MAP_MASK;
3551
3552                         if (!se_mask[idx]) {
3553                                 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
3554                         } else {
3555                                 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
3556                         }
3557                 }
3558
3559                 pkr0_mask &= rb_mask;
3560                 pkr1_mask &= rb_mask;
3561                 if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
3562                         raster_config_se &= ~PKR_MAP_MASK;
3563
3564                         if (!pkr0_mask) {
3565                                 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
3566                         } else {
3567                                 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
3568                         }
3569                 }
3570
3571                 if (rb_per_se >= 2) {
3572                         unsigned rb0_mask = 1 << (se * rb_per_se);
3573                         unsigned rb1_mask = rb0_mask << 1;
3574
3575                         rb0_mask &= rb_mask;
3576                         rb1_mask &= rb_mask;
3577                         if (!rb0_mask || !rb1_mask) {
3578                                 raster_config_se &= ~RB_MAP_PKR0_MASK;
3579
3580                                 if (!rb0_mask) {
3581                                         raster_config_se |=
3582                                                 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
3583                                 } else {
3584                                         raster_config_se |=
3585                                                 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
3586                                 }
3587                         }
3588
3589                         if (rb_per_se > 2) {
3590                                 rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
3591                                 rb1_mask = rb0_mask << 1;
3592                                 rb0_mask &= rb_mask;
3593                                 rb1_mask &= rb_mask;
3594                                 if (!rb0_mask || !rb1_mask) {
3595                                         raster_config_se &= ~RB_MAP_PKR1_MASK;
3596
3597                                         if (!rb0_mask) {
3598                                                 raster_config_se |=
3599                                                         RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
3600                                         } else {
3601                                                 raster_config_se |=
3602                                                         RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
3603                                         }
3604                                 }
3605                         }
3606                 }
3607
3608                 /* GRBM_GFX_INDEX has a different offset on VI */
3609                 gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
3610                 WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
3611                 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3612         }
3613
3614         /* GRBM_GFX_INDEX has a different offset on VI */
3615         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3616 }
3617
3618 static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
3619 {
3620         int i, j;
3621         u32 data;
3622         u32 raster_config = 0, raster_config_1 = 0;
3623         u32 active_rbs = 0;
3624         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
3625                                         adev->gfx.config.max_sh_per_se;
3626         unsigned num_rb_pipes;
3627
3628         mutex_lock(&adev->grbm_idx_mutex);
3629         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3630                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3631                         gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3632                         data = gfx_v8_0_get_rb_active_bitmap(adev);
3633                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
3634                                                rb_bitmap_width_per_sh);
3635                 }
3636         }
3637         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3638
3639         adev->gfx.config.backend_enable_mask = active_rbs;
3640         adev->gfx.config.num_rbs = hweight32(active_rbs);
3641
3642         num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
3643                              adev->gfx.config.max_shader_engines, 16);
3644
3645         gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1);
3646
3647         if (!adev->gfx.config.backend_enable_mask ||
3648                         adev->gfx.config.num_rbs >= num_rb_pipes) {
3649                 WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
3650                 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3651         } else {
3652                 gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
3653                                                         adev->gfx.config.backend_enable_mask,
3654                                                         num_rb_pipes);
3655         }
3656
3657         /* cache the values for userspace */
3658         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3659                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3660                         gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3661                         adev->gfx.config.rb_config[i][j].rb_backend_disable =
3662                                 RREG32(mmCC_RB_BACKEND_DISABLE);
3663                         adev->gfx.config.rb_config[i][j].user_rb_backend_disable =
3664                                 RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3665                         adev->gfx.config.rb_config[i][j].raster_config =
3666                                 RREG32(mmPA_SC_RASTER_CONFIG);
3667                         adev->gfx.config.rb_config[i][j].raster_config_1 =
3668                                 RREG32(mmPA_SC_RASTER_CONFIG_1);
3669                 }
3670         }
3671         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3672         mutex_unlock(&adev->grbm_idx_mutex);
3673 }
3674
3675 #define DEFAULT_SH_MEM_BASES    (0x6000)
3676 /**
3677  * gfx_v8_0_init_compute_vmid - gart enable
3678  *
3679  * @adev: amdgpu_device pointer
3680  *
3681  * Initialize compute vmid sh_mem registers
3682  *
3683  */
3684 static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
3685 {
3686         int i;
3687         uint32_t sh_mem_config;
3688         uint32_t sh_mem_bases;
3689
3690         /*
3691          * Configure apertures:
3692          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
3693          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
3694          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
3695          */
3696         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
3697
3698         sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 <<
3699                         SH_MEM_CONFIG__ADDRESS_MODE__SHIFT |
3700                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
3701                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
3702                         MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
3703                         SH_MEM_CONFIG__PRIVATE_ATC_MASK;
3704
3705         mutex_lock(&adev->srbm_mutex);
3706         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
3707                 vi_srbm_select(adev, 0, 0, 0, i);
3708                 /* CP and shaders */
3709                 WREG32(mmSH_MEM_CONFIG, sh_mem_config);
3710                 WREG32(mmSH_MEM_APE1_BASE, 1);
3711                 WREG32(mmSH_MEM_APE1_LIMIT, 0);
3712                 WREG32(mmSH_MEM_BASES, sh_mem_bases);
3713         }
3714         vi_srbm_select(adev, 0, 0, 0, 0);
3715         mutex_unlock(&adev->srbm_mutex);
3716
3717         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
3718            access. These should be enabled by FW for target VMIDs. */
3719         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
3720                 WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
3721                 WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
3722                 WREG32(amdgpu_gds_reg_offset[i].gws, 0);
3723                 WREG32(amdgpu_gds_reg_offset[i].oa, 0);
3724         }
3725 }
3726
3727 static void gfx_v8_0_init_gds_vmid(struct amdgpu_device *adev)
3728 {
3729         int vmid;
3730
3731         /*
3732          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
3733          * access. Compute VMIDs should be enabled by FW for target VMIDs,
3734          * the driver can enable them for graphics. VMID0 should maintain
3735          * access so that HWS firmware can save/restore entries.
3736          */
3737         for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
3738                 WREG32(amdgpu_gds_reg_offset[vmid].mem_base, 0);
3739                 WREG32(amdgpu_gds_reg_offset[vmid].mem_size, 0);
3740                 WREG32(amdgpu_gds_reg_offset[vmid].gws, 0);
3741                 WREG32(amdgpu_gds_reg_offset[vmid].oa, 0);
3742         }
3743 }
3744
3745 static void gfx_v8_0_config_init(struct amdgpu_device *adev)
3746 {
3747         switch (adev->asic_type) {
3748         default:
3749                 adev->gfx.config.double_offchip_lds_buf = 1;
3750                 break;
3751         case CHIP_CARRIZO:
3752         case CHIP_STONEY:
3753                 adev->gfx.config.double_offchip_lds_buf = 0;
3754                 break;
3755         }
3756 }
3757
3758 static void gfx_v8_0_constants_init(struct amdgpu_device *adev)
3759 {
3760         u32 tmp, sh_static_mem_cfg;
3761         int i;
3762
3763         WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF);
3764         WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3765         WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3766         WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
3767
3768         gfx_v8_0_tiling_mode_table_init(adev);
3769         gfx_v8_0_setup_rb(adev);
3770         gfx_v8_0_get_cu_info(adev);
3771         gfx_v8_0_config_init(adev);
3772
3773         /* XXX SH_MEM regs */
3774         /* where to put LDS, scratch, GPUVM in FSA64 space */
3775         sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG,
3776                                    SWIZZLE_ENABLE, 1);
3777         sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3778                                    ELEMENT_SIZE, 1);
3779         sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3780                                    INDEX_STRIDE, 3);
3781         WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
3782
3783         mutex_lock(&adev->srbm_mutex);
3784         for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
3785                 vi_srbm_select(adev, 0, 0, 0, i);
3786                 /* CP and shaders */
3787                 if (i == 0) {
3788                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC);
3789                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
3790                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
3791                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
3792                         WREG32(mmSH_MEM_CONFIG, tmp);
3793                         WREG32(mmSH_MEM_BASES, 0);
3794                 } else {
3795                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC);
3796                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
3797                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
3798                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
3799                         WREG32(mmSH_MEM_CONFIG, tmp);
3800                         tmp = adev->gmc.shared_aperture_start >> 48;
3801                         WREG32(mmSH_MEM_BASES, tmp);
3802                 }
3803
3804                 WREG32(mmSH_MEM_APE1_BASE, 1);
3805                 WREG32(mmSH_MEM_APE1_LIMIT, 0);
3806         }
3807         vi_srbm_select(adev, 0, 0, 0, 0);
3808         mutex_unlock(&adev->srbm_mutex);
3809
3810         gfx_v8_0_init_compute_vmid(adev);
3811         gfx_v8_0_init_gds_vmid(adev);
3812
3813         mutex_lock(&adev->grbm_idx_mutex);
3814         /*
3815          * making sure that the following register writes will be broadcasted
3816          * to all the shaders
3817          */
3818         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3819
3820         WREG32(mmPA_SC_FIFO_SIZE,
3821                    (adev->gfx.config.sc_prim_fifo_size_frontend <<
3822                         PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
3823                    (adev->gfx.config.sc_prim_fifo_size_backend <<
3824                         PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
3825                    (adev->gfx.config.sc_hiz_tile_fifo_size <<
3826                         PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
3827                    (adev->gfx.config.sc_earlyz_tile_fifo_size <<
3828                         PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT));
3829
3830         tmp = RREG32(mmSPI_ARB_PRIORITY);
3831         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2);
3832         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2);
3833         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2);
3834         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2);
3835         WREG32(mmSPI_ARB_PRIORITY, tmp);
3836
3837         mutex_unlock(&adev->grbm_idx_mutex);
3838
3839 }
3840
3841 static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
3842 {
3843         u32 i, j, k;
3844         u32 mask;
3845
3846         mutex_lock(&adev->grbm_idx_mutex);
3847         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3848                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3849                         gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3850                         for (k = 0; k < adev->usec_timeout; k++) {
3851                                 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)
3852                                         break;
3853                                 udelay(1);
3854                         }
3855                         if (k == adev->usec_timeout) {
3856                                 gfx_v8_0_select_se_sh(adev, 0xffffffff,
3857                                                       0xffffffff, 0xffffffff);
3858                                 mutex_unlock(&adev->grbm_idx_mutex);
3859                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
3860                                          i, j);
3861                                 return;
3862                         }
3863                 }
3864         }
3865         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3866         mutex_unlock(&adev->grbm_idx_mutex);
3867
3868         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
3869                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
3870                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
3871                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
3872         for (k = 0; k < adev->usec_timeout; k++) {
3873                 if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
3874                         break;
3875                 udelay(1);
3876         }
3877 }
3878
3879 static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
3880                                                bool enable)
3881 {
3882         u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
3883
3884         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
3885         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
3886         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
3887         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
3888
3889         WREG32(mmCP_INT_CNTL_RING0, tmp);
3890 }
3891
3892 static void gfx_v8_0_init_csb(struct amdgpu_device *adev)
3893 {
3894         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
3895         /* csib */
3896         WREG32(mmRLC_CSIB_ADDR_HI,
3897                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
3898         WREG32(mmRLC_CSIB_ADDR_LO,
3899                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
3900         WREG32(mmRLC_CSIB_LENGTH,
3901                         adev->gfx.rlc.clear_state_size);
3902 }
3903
3904 static void gfx_v8_0_parse_ind_reg_list(int *register_list_format,
3905                                 int ind_offset,
3906                                 int list_size,
3907                                 int *unique_indices,
3908                                 int *indices_count,
3909                                 int max_indices,
3910                                 int *ind_start_offsets,
3911                                 int *offset_count,
3912                                 int max_offset)
3913 {
3914         int indices;
3915         bool new_entry = true;
3916
3917         for (; ind_offset < list_size; ind_offset++) {
3918
3919                 if (new_entry) {
3920                         new_entry = false;
3921                         ind_start_offsets[*offset_count] = ind_offset;
3922                         *offset_count = *offset_count + 1;
3923                         BUG_ON(*offset_count >= max_offset);
3924                 }
3925
3926                 if (register_list_format[ind_offset] == 0xFFFFFFFF) {
3927                         new_entry = true;
3928                         continue;
3929                 }
3930
3931                 ind_offset += 2;
3932
3933                 /* look for the matching indice */
3934                 for (indices = 0;
3935                         indices < *indices_count;
3936                         indices++) {
3937                         if (unique_indices[indices] ==
3938                                 register_list_format[ind_offset])
3939                                 break;
3940                 }
3941
3942                 if (indices >= *indices_count) {
3943                         unique_indices[*indices_count] =
3944                                 register_list_format[ind_offset];
3945                         indices = *indices_count;
3946                         *indices_count = *indices_count + 1;
3947                         BUG_ON(*indices_count >= max_indices);
3948                 }
3949
3950                 register_list_format[ind_offset] = indices;
3951         }
3952 }
3953
3954 static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
3955 {
3956         int i, temp, data;
3957         int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0};
3958         int indices_count = 0;
3959         int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
3960         int offset_count = 0;
3961
3962         int list_size;
3963         unsigned int *register_list_format =
3964                 kmemdup(adev->gfx.rlc.register_list_format,
3965                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
3966         if (!register_list_format)
3967                 return -ENOMEM;
3968
3969         gfx_v8_0_parse_ind_reg_list(register_list_format,
3970                                 RLC_FormatDirectRegListLength,
3971                                 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
3972                                 unique_indices,
3973                                 &indices_count,
3974                                 ARRAY_SIZE(unique_indices),
3975                                 indirect_start_offsets,
3976                                 &offset_count,
3977                                 ARRAY_SIZE(indirect_start_offsets));
3978
3979         /* save and restore list */
3980         WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1);
3981
3982         WREG32(mmRLC_SRM_ARAM_ADDR, 0);
3983         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
3984                 WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]);
3985
3986         /* indirect list */
3987         WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start);
3988         for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++)
3989                 WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]);
3990
3991         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
3992         list_size = list_size >> 1;
3993         WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size);
3994         WREG32(mmRLC_GPM_SCRATCH_DATA, list_size);
3995
3996         /* starting offsets starts */
3997         WREG32(mmRLC_GPM_SCRATCH_ADDR,
3998                 adev->gfx.rlc.starting_offsets_start);
3999         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
4000                 WREG32(mmRLC_GPM_SCRATCH_DATA,
4001                                 indirect_start_offsets[i]);
4002
4003         /* unique indices */
4004         temp = mmRLC_SRM_INDEX_CNTL_ADDR_0;
4005         data = mmRLC_SRM_INDEX_CNTL_DATA_0;
4006         for (i = 0; i < ARRAY_SIZE(unique_indices); i++) {
4007                 if (unique_indices[i] != 0) {
4008                         WREG32(temp + i, unique_indices[i] & 0x3FFFF);
4009                         WREG32(data + i, unique_indices[i] >> 20);
4010                 }
4011         }
4012         kfree(register_list_format);
4013
4014         return 0;
4015 }
4016
4017 static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev)
4018 {
4019         WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1);
4020 }
4021
4022 static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)
4023 {
4024         uint32_t data;
4025
4026         WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60);
4027
4028         data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10);
4029         data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10);
4030         data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10);
4031         data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10);
4032         WREG32(mmRLC_PG_DELAY, data);
4033
4034         WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3);
4035         WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0);
4036
4037 }
4038
4039 static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
4040                                                 bool enable)
4041 {
4042         WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0);
4043 }
4044
4045 static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
4046                                                   bool enable)
4047 {
4048         WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0);
4049 }
4050
4051 static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable)
4052 {
4053         WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1);
4054 }
4055
4056 static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
4057 {
4058         if ((adev->asic_type == CHIP_CARRIZO) ||
4059             (adev->asic_type == CHIP_STONEY)) {
4060                 gfx_v8_0_init_csb(adev);
4061                 gfx_v8_0_init_save_restore_list(adev);
4062                 gfx_v8_0_enable_save_restore_machine(adev);
4063                 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);
4064                 gfx_v8_0_init_power_gating(adev);
4065                 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
4066         } else if ((adev->asic_type == CHIP_POLARIS11) ||
4067                    (adev->asic_type == CHIP_POLARIS12) ||
4068                    (adev->asic_type == CHIP_VEGAM)) {
4069                 gfx_v8_0_init_csb(adev);
4070                 gfx_v8_0_init_save_restore_list(adev);
4071                 gfx_v8_0_enable_save_restore_machine(adev);
4072                 gfx_v8_0_init_power_gating(adev);
4073         }
4074
4075 }
4076
4077 static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev)
4078 {
4079         WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0);
4080
4081         gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4082         gfx_v8_0_wait_for_rlc_serdes(adev);
4083 }
4084
4085 static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev)
4086 {
4087         WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4088         udelay(50);
4089
4090         WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
4091         udelay(50);
4092 }
4093
4094 static void gfx_v8_0_rlc_start(struct amdgpu_device *adev)
4095 {
4096         WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1);
4097
4098         /* carrizo do enable cp interrupt after cp inited */
4099         if (!(adev->flags & AMD_IS_APU))
4100                 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4101
4102         udelay(50);
4103 }
4104
4105 static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
4106 {
4107         if (amdgpu_sriov_vf(adev)) {
4108                 gfx_v8_0_init_csb(adev);
4109                 return 0;
4110         }
4111
4112         adev->gfx.rlc.funcs->stop(adev);
4113         adev->gfx.rlc.funcs->reset(adev);
4114         gfx_v8_0_init_pg(adev);
4115         adev->gfx.rlc.funcs->start(adev);
4116
4117         return 0;
4118 }
4119
4120 static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
4121 {
4122         u32 tmp = RREG32(mmCP_ME_CNTL);
4123
4124         if (enable) {
4125                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0);
4126                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0);
4127                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0);
4128         } else {
4129                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1);
4130                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1);
4131                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1);
4132         }
4133         WREG32(mmCP_ME_CNTL, tmp);
4134         udelay(50);
4135 }
4136
4137 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev)
4138 {
4139         u32 count = 0;
4140         const struct cs_section_def *sect = NULL;
4141         const struct cs_extent_def *ext = NULL;
4142
4143         /* begin clear state */
4144         count += 2;
4145         /* context control state */
4146         count += 3;
4147
4148         for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4149                 for (ext = sect->section; ext->extent != NULL; ++ext) {
4150                         if (sect->id == SECT_CONTEXT)
4151                                 count += 2 + ext->reg_count;
4152                         else
4153                                 return 0;
4154                 }
4155         }
4156         /* pa_sc_raster_config/pa_sc_raster_config1 */
4157         count += 4;
4158         /* end clear state */
4159         count += 2;
4160         /* clear state */
4161         count += 2;
4162
4163         return count;
4164 }
4165
4166 static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)
4167 {
4168         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
4169         const struct cs_section_def *sect = NULL;
4170         const struct cs_extent_def *ext = NULL;
4171         int r, i;
4172
4173         /* init the CP */
4174         WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
4175         WREG32(mmCP_ENDIAN_SWAP, 0);
4176         WREG32(mmCP_DEVICE_ID, 1);
4177
4178         gfx_v8_0_cp_gfx_enable(adev, true);
4179
4180         r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4);
4181         if (r) {
4182                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
4183                 return r;
4184         }
4185
4186         /* clear state buffer */
4187         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4188         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4189
4190         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4191         amdgpu_ring_write(ring, 0x80000000);
4192         amdgpu_ring_write(ring, 0x80000000);
4193
4194         for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4195                 for (ext = sect->section; ext->extent != NULL; ++ext) {
4196                         if (sect->id == SECT_CONTEXT) {
4197                                 amdgpu_ring_write(ring,
4198                                        PACKET3(PACKET3_SET_CONTEXT_REG,
4199                                                ext->reg_count));
4200                                 amdgpu_ring_write(ring,
4201                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
4202                                 for (i = 0; i < ext->reg_count; i++)
4203                                         amdgpu_ring_write(ring, ext->extent[i]);
4204                         }
4205                 }
4206         }
4207
4208         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4209         amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
4210         amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
4211         amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
4212
4213         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4214         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4215
4216         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4217         amdgpu_ring_write(ring, 0);
4218
4219         /* init the CE partitions */
4220         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4221         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4222         amdgpu_ring_write(ring, 0x8000);
4223         amdgpu_ring_write(ring, 0x8000);
4224
4225         amdgpu_ring_commit(ring);
4226
4227         return 0;
4228 }
4229 static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring)
4230 {
4231         u32 tmp;
4232         /* no gfx doorbells on iceland */
4233         if (adev->asic_type == CHIP_TOPAZ)
4234                 return;
4235
4236         tmp = RREG32(mmCP_RB_DOORBELL_CONTROL);
4237
4238         if (ring->use_doorbell) {
4239                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4240                                 DOORBELL_OFFSET, ring->doorbell_index);
4241                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4242                                                 DOORBELL_HIT, 0);
4243                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4244                                             DOORBELL_EN, 1);
4245         } else {
4246                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
4247         }
4248
4249         WREG32(mmCP_RB_DOORBELL_CONTROL, tmp);
4250
4251         if (adev->flags & AMD_IS_APU)
4252                 return;
4253
4254         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
4255                                         DOORBELL_RANGE_LOWER,
4256                                         adev->doorbell_index.gfx_ring0);
4257         WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
4258
4259         WREG32(mmCP_RB_DOORBELL_RANGE_UPPER,
4260                 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
4261 }
4262
4263 static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)
4264 {
4265         struct amdgpu_ring *ring;
4266         u32 tmp;
4267         u32 rb_bufsz;
4268         u64 rb_addr, rptr_addr, wptr_gpu_addr;
4269
4270         /* Set the write pointer delay */
4271         WREG32(mmCP_RB_WPTR_DELAY, 0);
4272
4273         /* set the RB to use vmid 0 */
4274         WREG32(mmCP_RB_VMID, 0);
4275
4276         /* Set ring buffer size */
4277         ring = &adev->gfx.gfx_ring[0];
4278         rb_bufsz = order_base_2(ring->ring_size / 8);
4279         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
4280         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
4281         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3);
4282         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1);
4283 #ifdef __BIG_ENDIAN
4284         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
4285 #endif
4286         WREG32(mmCP_RB0_CNTL, tmp);
4287
4288         /* Initialize the ring buffer's read and write pointers */
4289         WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
4290         ring->wptr = 0;
4291         WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
4292
4293         /* set the wb address wether it's enabled or not */
4294         rptr_addr = ring->rptr_gpu_addr;
4295         WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
4296         WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
4297
4298         wptr_gpu_addr = ring->wptr_gpu_addr;
4299         WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
4300         WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
4301         mdelay(1);
4302         WREG32(mmCP_RB0_CNTL, tmp);
4303
4304         rb_addr = ring->gpu_addr >> 8;
4305         WREG32(mmCP_RB0_BASE, rb_addr);
4306         WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
4307
4308         gfx_v8_0_set_cpg_door_bell(adev, ring);
4309         /* start the ring */
4310         amdgpu_ring_clear_ring(ring);
4311         gfx_v8_0_cp_gfx_start(adev);
4312         ring->sched.ready = true;
4313
4314         return 0;
4315 }
4316
4317 static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
4318 {
4319         if (enable) {
4320                 WREG32(mmCP_MEC_CNTL, 0);
4321         } else {
4322                 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
4323                 adev->gfx.kiq.ring.sched.ready = false;
4324         }
4325         udelay(50);
4326 }
4327
4328 /* KIQ functions */
4329 static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring)
4330 {
4331         uint32_t tmp;
4332         struct amdgpu_device *adev = ring->adev;
4333
4334         /* tell RLC which is KIQ queue */
4335         tmp = RREG32(mmRLC_CP_SCHEDULERS);
4336         tmp &= 0xffffff00;
4337         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
4338         WREG32(mmRLC_CP_SCHEDULERS, tmp);
4339         tmp |= 0x80;
4340         WREG32(mmRLC_CP_SCHEDULERS, tmp);
4341 }
4342
4343 static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev)
4344 {
4345         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
4346         uint64_t queue_mask = 0;
4347         int r, i;
4348
4349         for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
4350                 if (!test_bit(i, adev->gfx.mec.queue_bitmap))
4351                         continue;
4352
4353                 /* This situation may be hit in the future if a new HW
4354                  * generation exposes more than 64 queues. If so, the
4355                  * definition of queue_mask needs updating */
4356                 if (WARN_ON(i >= (sizeof(queue_mask)*8))) {
4357                         DRM_ERROR("Invalid KCQ enabled: %d\n", i);
4358                         break;
4359                 }
4360
4361                 queue_mask |= (1ull << i);
4362         }
4363
4364         r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8);
4365         if (r) {
4366                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
4367                 return r;
4368         }
4369         /* set resources */
4370         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
4371         amdgpu_ring_write(kiq_ring, 0); /* vmid_mask:0 queue_type:0 (KIQ) */
4372         amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
4373         amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
4374         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
4375         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
4376         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
4377         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
4378         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4379                 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4380                 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
4381                 uint64_t wptr_addr = ring->wptr_gpu_addr;
4382
4383                 /* map queues */
4384                 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
4385                 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
4386                 amdgpu_ring_write(kiq_ring,
4387                                   PACKET3_MAP_QUEUES_NUM_QUEUES(1));
4388                 amdgpu_ring_write(kiq_ring,
4389                                   PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) |
4390                                   PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
4391                                   PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
4392                                   PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */
4393                 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
4394                 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
4395                 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
4396                 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
4397         }
4398
4399         amdgpu_ring_commit(kiq_ring);
4400
4401         return 0;
4402 }
4403
4404 static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req)
4405 {
4406         int i, r = 0;
4407
4408         if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) {
4409                 WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, req);
4410                 for (i = 0; i < adev->usec_timeout; i++) {
4411                         if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK))
4412                                 break;
4413                         udelay(1);
4414                 }
4415                 if (i == adev->usec_timeout)
4416                         r = -ETIMEDOUT;
4417         }
4418         WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);
4419         WREG32(mmCP_HQD_PQ_RPTR, 0);
4420         WREG32(mmCP_HQD_PQ_WPTR, 0);
4421
4422         return r;
4423 }
4424
4425 static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *mqd)
4426 {
4427         struct amdgpu_device *adev = ring->adev;
4428
4429         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
4430                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
4431                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
4432                         mqd->cp_hqd_queue_priority =
4433                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
4434                 }
4435         }
4436 }
4437
4438 static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
4439 {
4440         struct amdgpu_device *adev = ring->adev;
4441         struct vi_mqd *mqd = ring->mqd_ptr;
4442         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4443         uint32_t tmp;
4444
4445         mqd->header = 0xC0310800;
4446         mqd->compute_pipelinestat_enable = 0x00000001;
4447         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4448         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4449         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4450         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4451         mqd->compute_misc_reserved = 0x00000003;
4452         mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr
4453                                                      + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
4454         mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr
4455                                                      + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
4456         eop_base_addr = ring->eop_gpu_addr >> 8;
4457         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4458         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4459
4460         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4461         tmp = RREG32(mmCP_HQD_EOP_CONTROL);
4462         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4463                         (order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1));
4464
4465         mqd->cp_hqd_eop_control = tmp;
4466
4467         /* enable doorbell? */
4468         tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL),
4469                             CP_HQD_PQ_DOORBELL_CONTROL,
4470                             DOORBELL_EN,
4471                             ring->use_doorbell ? 1 : 0);
4472
4473         mqd->cp_hqd_pq_doorbell_control = tmp;
4474
4475         /* set the pointer to the MQD */
4476         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
4477         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
4478
4479         /* set MQD vmid to 0 */
4480         tmp = RREG32(mmCP_MQD_CONTROL);
4481         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4482         mqd->cp_mqd_control = tmp;
4483
4484         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4485         hqd_gpu_addr = ring->gpu_addr >> 8;
4486         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4487         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4488
4489         /* set up the HQD, this is similar to CP_RB0_CNTL */
4490         tmp = RREG32(mmCP_HQD_PQ_CONTROL);
4491         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4492                             (order_base_2(ring->ring_size / 4) - 1));
4493         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4494                         (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
4495 #ifdef __BIG_ENDIAN
4496         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
4497 #endif
4498         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
4499         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
4500         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4501         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4502         mqd->cp_hqd_pq_control = tmp;
4503
4504         /* set the wb address whether it's enabled or not */
4505         wb_gpu_addr = ring->rptr_gpu_addr;
4506         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4507         mqd->cp_hqd_pq_rptr_report_addr_hi =
4508                 upper_32_bits(wb_gpu_addr) & 0xffff;
4509
4510         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4511         wb_gpu_addr = ring->wptr_gpu_addr;
4512         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4513         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4514
4515         tmp = 0;
4516         /* enable the doorbell if requested */
4517         if (ring->use_doorbell) {
4518                 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
4519                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4520                                 DOORBELL_OFFSET, ring->doorbell_index);
4521
4522                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4523                                          DOORBELL_EN, 1);
4524                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4525                                          DOORBELL_SOURCE, 0);
4526                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4527                                          DOORBELL_HIT, 0);
4528         }
4529
4530         mqd->cp_hqd_pq_doorbell_control = tmp;
4531
4532         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4533         ring->wptr = 0;
4534         mqd->cp_hqd_pq_wptr = ring->wptr;
4535         mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
4536
4537         /* set the vmid for the queue */
4538         mqd->cp_hqd_vmid = 0;
4539
4540         tmp = RREG32(mmCP_HQD_PERSISTENT_STATE);
4541         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
4542         mqd->cp_hqd_persistent_state = tmp;
4543
4544         /* set MTYPE */
4545         tmp = RREG32(mmCP_HQD_IB_CONTROL);
4546         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4547         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MTYPE, 3);
4548         mqd->cp_hqd_ib_control = tmp;
4549
4550         tmp = RREG32(mmCP_HQD_IQ_TIMER);
4551         tmp = REG_SET_FIELD(tmp, CP_HQD_IQ_TIMER, MTYPE, 3);
4552         mqd->cp_hqd_iq_timer = tmp;
4553
4554         tmp = RREG32(mmCP_HQD_CTX_SAVE_CONTROL);
4555         tmp = REG_SET_FIELD(tmp, CP_HQD_CTX_SAVE_CONTROL, MTYPE, 3);
4556         mqd->cp_hqd_ctx_save_control = tmp;
4557
4558         /* defaults */
4559         mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR);
4560         mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR);
4561         mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO);
4562         mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI);
4563         mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET);
4564         mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE);
4565         mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET);
4566         mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE);
4567         mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS);
4568         mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR);
4569         mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM);
4570         mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES);
4571
4572         /* set static priority for a queue/ring */
4573         gfx_v8_0_mqd_set_priority(ring, mqd);
4574         mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
4575
4576         /* map_queues packet doesn't need activate the queue,
4577          * so only kiq need set this field.
4578          */
4579         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
4580                 mqd->cp_hqd_active = 1;
4581
4582         return 0;
4583 }
4584
4585 static int gfx_v8_0_mqd_commit(struct amdgpu_device *adev,
4586                         struct vi_mqd *mqd)
4587 {
4588         uint32_t mqd_reg;
4589         uint32_t *mqd_data;
4590
4591         /* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_HQD_ERROR */
4592         mqd_data = &mqd->cp_mqd_base_addr_lo;
4593
4594         /* disable wptr polling */
4595         WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0);
4596
4597         /* program all HQD registers */
4598         for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_HQD_EOP_CONTROL; mqd_reg++)
4599                 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4600
4601         /* Tonga errata: EOP RPTR/WPTR should be left unmodified.
4602          * This is safe since EOP RPTR==WPTR for any inactive HQD
4603          * on ASICs that do not support context-save.
4604          * EOP writes/reads can start anywhere in the ring.
4605          */
4606         if (adev->asic_type != CHIP_TONGA) {
4607                 WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr);
4608                 WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr);
4609                 WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem);
4610         }
4611
4612         for (mqd_reg = mmCP_HQD_EOP_EVENTS; mqd_reg <= mmCP_HQD_ERROR; mqd_reg++)
4613                 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4614
4615         /* activate the HQD */
4616         for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++)
4617                 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4618
4619         return 0;
4620 }
4621
4622 static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
4623 {
4624         struct amdgpu_device *adev = ring->adev;
4625         struct vi_mqd *mqd = ring->mqd_ptr;
4626         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
4627
4628         gfx_v8_0_kiq_setting(ring);
4629
4630         if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4631                 /* reset MQD to a clean status */
4632                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4633                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
4634
4635                 /* reset ring buffer */
4636                 ring->wptr = 0;
4637                 amdgpu_ring_clear_ring(ring);
4638                 mutex_lock(&adev->srbm_mutex);
4639                 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4640                 gfx_v8_0_mqd_commit(adev, mqd);
4641                 vi_srbm_select(adev, 0, 0, 0, 0);
4642                 mutex_unlock(&adev->srbm_mutex);
4643         } else {
4644                 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4645                 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4646                 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
4647                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4648                         amdgpu_ring_clear_ring(ring);
4649                 mutex_lock(&adev->srbm_mutex);
4650                 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4651                 gfx_v8_0_mqd_init(ring);
4652                 gfx_v8_0_mqd_commit(adev, mqd);
4653                 vi_srbm_select(adev, 0, 0, 0, 0);
4654                 mutex_unlock(&adev->srbm_mutex);
4655
4656                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4657                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
4658         }
4659
4660         return 0;
4661 }
4662
4663 static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
4664 {
4665         struct amdgpu_device *adev = ring->adev;
4666         struct vi_mqd *mqd = ring->mqd_ptr;
4667         int mqd_idx = ring - &adev->gfx.compute_ring[0];
4668
4669         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4670                 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4671                 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4672                 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
4673                 mutex_lock(&adev->srbm_mutex);
4674                 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4675                 gfx_v8_0_mqd_init(ring);
4676                 vi_srbm_select(adev, 0, 0, 0, 0);
4677                 mutex_unlock(&adev->srbm_mutex);
4678
4679                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4680                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
4681         } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4682                 /* reset MQD to a clean status */
4683                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4684                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
4685                 /* reset ring buffer */
4686                 ring->wptr = 0;
4687                 amdgpu_ring_clear_ring(ring);
4688         } else {
4689                 amdgpu_ring_clear_ring(ring);
4690         }
4691         return 0;
4692 }
4693
4694 static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev)
4695 {
4696         if (adev->asic_type > CHIP_TONGA) {
4697                 WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, adev->doorbell_index.kiq << 2);
4698                 WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, adev->doorbell_index.mec_ring7 << 2);
4699         }
4700         /* enable doorbells */
4701         WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4702 }
4703
4704 static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)
4705 {
4706         struct amdgpu_ring *ring;
4707         int r;
4708
4709         ring = &adev->gfx.kiq.ring;
4710
4711         r = amdgpu_bo_reserve(ring->mqd_obj, false);
4712         if (unlikely(r != 0))
4713                 return r;
4714
4715         r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr);
4716         if (unlikely(r != 0))
4717                 return r;
4718
4719         gfx_v8_0_kiq_init_queue(ring);
4720         amdgpu_bo_kunmap(ring->mqd_obj);
4721         ring->mqd_ptr = NULL;
4722         amdgpu_bo_unreserve(ring->mqd_obj);
4723         ring->sched.ready = true;
4724         return 0;
4725 }
4726
4727 static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev)
4728 {
4729         struct amdgpu_ring *ring = NULL;
4730         int r = 0, i;
4731
4732         gfx_v8_0_cp_compute_enable(adev, true);
4733
4734         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4735                 ring = &adev->gfx.compute_ring[i];
4736
4737                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4738                 if (unlikely(r != 0))
4739                         goto done;
4740                 r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr);
4741                 if (!r) {
4742                         r = gfx_v8_0_kcq_init_queue(ring);
4743                         amdgpu_bo_kunmap(ring->mqd_obj);
4744                         ring->mqd_ptr = NULL;
4745                 }
4746                 amdgpu_bo_unreserve(ring->mqd_obj);
4747                 if (r)
4748                         goto done;
4749         }
4750
4751         gfx_v8_0_set_mec_doorbell_range(adev);
4752
4753         r = gfx_v8_0_kiq_kcq_enable(adev);
4754         if (r)
4755                 goto done;
4756
4757 done:
4758         return r;
4759 }
4760
4761 static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev)
4762 {
4763         int r, i;
4764         struct amdgpu_ring *ring;
4765
4766         /* collect all the ring_tests here, gfx, kiq, compute */
4767         ring = &adev->gfx.gfx_ring[0];
4768         r = amdgpu_ring_test_helper(ring);
4769         if (r)
4770                 return r;
4771
4772         ring = &adev->gfx.kiq.ring;
4773         r = amdgpu_ring_test_helper(ring);
4774         if (r)
4775                 return r;
4776
4777         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4778                 ring = &adev->gfx.compute_ring[i];
4779                 amdgpu_ring_test_helper(ring);
4780         }
4781
4782         return 0;
4783 }
4784
4785 static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
4786 {
4787         int r;
4788
4789         if (!(adev->flags & AMD_IS_APU))
4790                 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4791
4792         r = gfx_v8_0_kiq_resume(adev);
4793         if (r)
4794                 return r;
4795
4796         r = gfx_v8_0_cp_gfx_resume(adev);
4797         if (r)
4798                 return r;
4799
4800         r = gfx_v8_0_kcq_resume(adev);
4801         if (r)
4802                 return r;
4803
4804         r = gfx_v8_0_cp_test_all_rings(adev);
4805         if (r)
4806                 return r;
4807
4808         gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4809
4810         return 0;
4811 }
4812
4813 static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable)
4814 {
4815         gfx_v8_0_cp_gfx_enable(adev, enable);
4816         gfx_v8_0_cp_compute_enable(adev, enable);
4817 }
4818
4819 static int gfx_v8_0_hw_init(void *handle)
4820 {
4821         int r;
4822         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4823
4824         gfx_v8_0_init_golden_registers(adev);
4825         gfx_v8_0_constants_init(adev);
4826
4827         r = adev->gfx.rlc.funcs->resume(adev);
4828         if (r)
4829                 return r;
4830
4831         r = gfx_v8_0_cp_resume(adev);
4832
4833         return r;
4834 }
4835
4836 static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev)
4837 {
4838         int r, i;
4839         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
4840
4841         r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings);
4842         if (r)
4843                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
4844
4845         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4846                 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4847
4848                 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
4849                 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
4850                                                 PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */
4851                                                 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
4852                                                 PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |
4853                                                 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
4854                 amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
4855                 amdgpu_ring_write(kiq_ring, 0);
4856                 amdgpu_ring_write(kiq_ring, 0);
4857                 amdgpu_ring_write(kiq_ring, 0);
4858         }
4859         r = amdgpu_ring_test_helper(kiq_ring);
4860         if (r)
4861                 DRM_ERROR("KCQ disable failed\n");
4862
4863         return r;
4864 }
4865
4866 static bool gfx_v8_0_is_idle(void *handle)
4867 {
4868         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4869
4870         if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE)
4871                 || RREG32(mmGRBM_STATUS2) != 0x8)
4872                 return false;
4873         else
4874                 return true;
4875 }
4876
4877 static bool gfx_v8_0_rlc_is_idle(void *handle)
4878 {
4879         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4880
4881         if (RREG32(mmGRBM_STATUS2) != 0x8)
4882                 return false;
4883         else
4884                 return true;
4885 }
4886
4887 static int gfx_v8_0_wait_for_rlc_idle(void *handle)
4888 {
4889         unsigned int i;
4890         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4891
4892         for (i = 0; i < adev->usec_timeout; i++) {
4893                 if (gfx_v8_0_rlc_is_idle(handle))
4894                         return 0;
4895
4896                 udelay(1);
4897         }
4898         return -ETIMEDOUT;
4899 }
4900
4901 static int gfx_v8_0_wait_for_idle(void *handle)
4902 {
4903         unsigned int i;
4904         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4905
4906         for (i = 0; i < adev->usec_timeout; i++) {
4907                 if (gfx_v8_0_is_idle(handle))
4908                         return 0;
4909
4910                 udelay(1);
4911         }
4912         return -ETIMEDOUT;
4913 }
4914
4915 static int gfx_v8_0_hw_fini(void *handle)
4916 {
4917         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4918
4919         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4920         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4921
4922         amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4923
4924         amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0);
4925
4926         /* disable KCQ to avoid CPC touch memory not valid anymore */
4927         gfx_v8_0_kcq_disable(adev);
4928
4929         if (amdgpu_sriov_vf(adev)) {
4930                 pr_debug("For SRIOV client, shouldn't do anything.\n");
4931                 return 0;
4932         }
4933         amdgpu_gfx_rlc_enter_safe_mode(adev);
4934         if (!gfx_v8_0_wait_for_idle(adev))
4935                 gfx_v8_0_cp_enable(adev, false);
4936         else
4937                 pr_err("cp is busy, skip halt cp\n");
4938         if (!gfx_v8_0_wait_for_rlc_idle(adev))
4939                 adev->gfx.rlc.funcs->stop(adev);
4940         else
4941                 pr_err("rlc is busy, skip halt rlc\n");
4942         amdgpu_gfx_rlc_exit_safe_mode(adev);
4943
4944         return 0;
4945 }
4946
4947 static int gfx_v8_0_suspend(void *handle)
4948 {
4949         return gfx_v8_0_hw_fini(handle);
4950 }
4951
4952 static int gfx_v8_0_resume(void *handle)
4953 {
4954         return gfx_v8_0_hw_init(handle);
4955 }
4956
4957 static bool gfx_v8_0_check_soft_reset(void *handle)
4958 {
4959         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4960         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4961         u32 tmp;
4962
4963         /* GRBM_STATUS */
4964         tmp = RREG32(mmGRBM_STATUS);
4965         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4966                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4967                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4968                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4969                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4970                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK |
4971                    GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4972                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4973                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4974                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4975                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4976                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
4977                                                 SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
4978         }
4979
4980         /* GRBM_STATUS2 */
4981         tmp = RREG32(mmGRBM_STATUS2);
4982         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4983                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4984                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4985
4986         if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) ||
4987             REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) ||
4988             REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) {
4989                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4990                                                 SOFT_RESET_CPF, 1);
4991                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4992                                                 SOFT_RESET_CPC, 1);
4993                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4994                                                 SOFT_RESET_CPG, 1);
4995                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
4996                                                 SOFT_RESET_GRBM, 1);
4997         }
4998
4999         /* SRBM_STATUS */
5000         tmp = RREG32(mmSRBM_STATUS);
5001         if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING))
5002                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
5003                                                 SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
5004         if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY))
5005                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
5006                                                 SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);
5007
5008         if (grbm_soft_reset || srbm_soft_reset) {
5009                 adev->gfx.grbm_soft_reset = grbm_soft_reset;
5010                 adev->gfx.srbm_soft_reset = srbm_soft_reset;
5011                 return true;
5012         } else {
5013                 adev->gfx.grbm_soft_reset = 0;
5014                 adev->gfx.srbm_soft_reset = 0;
5015                 return false;
5016         }
5017 }
5018
5019 static int gfx_v8_0_pre_soft_reset(void *handle)
5020 {
5021         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5022         u32 grbm_soft_reset = 0;
5023
5024         if ((!adev->gfx.grbm_soft_reset) &&
5025             (!adev->gfx.srbm_soft_reset))
5026                 return 0;
5027
5028         grbm_soft_reset = adev->gfx.grbm_soft_reset;
5029
5030         /* stop the rlc */
5031         adev->gfx.rlc.funcs->stop(adev);
5032
5033         if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5034             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
5035                 /* Disable GFX parsing/prefetching */
5036                 gfx_v8_0_cp_gfx_enable(adev, false);
5037
5038         if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5039             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
5040             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
5041             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
5042                 int i;
5043
5044                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5045                         struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5046
5047                         mutex_lock(&adev->srbm_mutex);
5048                         vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5049                         gfx_v8_0_deactivate_hqd(adev, 2);
5050                         vi_srbm_select(adev, 0, 0, 0, 0);
5051                         mutex_unlock(&adev->srbm_mutex);
5052                 }
5053                 /* Disable MEC parsing/prefetching */
5054                 gfx_v8_0_cp_compute_enable(adev, false);
5055         }
5056
5057         return 0;
5058 }
5059
5060 static int gfx_v8_0_soft_reset(void *handle)
5061 {
5062         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5063         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5064         u32 tmp;
5065
5066         if ((!adev->gfx.grbm_soft_reset) &&
5067             (!adev->gfx.srbm_soft_reset))
5068                 return 0;
5069
5070         grbm_soft_reset = adev->gfx.grbm_soft_reset;
5071         srbm_soft_reset = adev->gfx.srbm_soft_reset;
5072
5073         if (grbm_soft_reset || srbm_soft_reset) {
5074                 tmp = RREG32(mmGMCON_DEBUG);
5075                 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1);
5076                 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1);
5077                 WREG32(mmGMCON_DEBUG, tmp);
5078                 udelay(50);
5079         }
5080
5081         if (grbm_soft_reset) {
5082                 tmp = RREG32(mmGRBM_SOFT_RESET);
5083                 tmp |= grbm_soft_reset;
5084                 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5085                 WREG32(mmGRBM_SOFT_RESET, tmp);
5086                 tmp = RREG32(mmGRBM_SOFT_RESET);
5087
5088                 udelay(50);
5089
5090                 tmp &= ~grbm_soft_reset;
5091                 WREG32(mmGRBM_SOFT_RESET, tmp);
5092                 tmp = RREG32(mmGRBM_SOFT_RESET);
5093         }
5094
5095         if (srbm_soft_reset) {
5096                 tmp = RREG32(mmSRBM_SOFT_RESET);
5097                 tmp |= srbm_soft_reset;
5098                 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5099                 WREG32(mmSRBM_SOFT_RESET, tmp);
5100                 tmp = RREG32(mmSRBM_SOFT_RESET);
5101
5102                 udelay(50);
5103
5104                 tmp &= ~srbm_soft_reset;
5105                 WREG32(mmSRBM_SOFT_RESET, tmp);
5106                 tmp = RREG32(mmSRBM_SOFT_RESET);
5107         }
5108
5109         if (grbm_soft_reset || srbm_soft_reset) {
5110                 tmp = RREG32(mmGMCON_DEBUG);
5111                 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0);
5112                 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0);
5113                 WREG32(mmGMCON_DEBUG, tmp);
5114         }
5115
5116         /* Wait a little for things to settle down */
5117         udelay(50);
5118
5119         return 0;
5120 }
5121
5122 static int gfx_v8_0_post_soft_reset(void *handle)
5123 {
5124         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5125         u32 grbm_soft_reset = 0;
5126
5127         if ((!adev->gfx.grbm_soft_reset) &&
5128             (!adev->gfx.srbm_soft_reset))
5129                 return 0;
5130
5131         grbm_soft_reset = adev->gfx.grbm_soft_reset;
5132
5133         if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5134             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
5135             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
5136             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
5137                 int i;
5138
5139                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5140                         struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5141
5142                         mutex_lock(&adev->srbm_mutex);
5143                         vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5144                         gfx_v8_0_deactivate_hqd(adev, 2);
5145                         vi_srbm_select(adev, 0, 0, 0, 0);
5146                         mutex_unlock(&adev->srbm_mutex);
5147                 }
5148                 gfx_v8_0_kiq_resume(adev);
5149                 gfx_v8_0_kcq_resume(adev);
5150         }
5151
5152         if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5153             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
5154                 gfx_v8_0_cp_gfx_resume(adev);
5155
5156         gfx_v8_0_cp_test_all_rings(adev);
5157
5158         adev->gfx.rlc.funcs->start(adev);
5159
5160         return 0;
5161 }
5162
5163 /**
5164  * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot
5165  *
5166  * @adev: amdgpu_device pointer
5167  *
5168  * Fetches a GPU clock counter snapshot.
5169  * Returns the 64 bit clock counter snapshot.
5170  */
5171 static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev)
5172 {
5173         uint64_t clock;
5174
5175         mutex_lock(&adev->gfx.gpu_clock_mutex);
5176         WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5177         clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |
5178                 ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5179         mutex_unlock(&adev->gfx.gpu_clock_mutex);
5180         return clock;
5181 }
5182
5183 static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
5184                                           uint32_t vmid,
5185                                           uint32_t gds_base, uint32_t gds_size,
5186                                           uint32_t gws_base, uint32_t gws_size,
5187                                           uint32_t oa_base, uint32_t oa_size)
5188 {
5189         /* GDS Base */
5190         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5191         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5192                                 WRITE_DATA_DST_SEL(0)));
5193         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base);
5194         amdgpu_ring_write(ring, 0);
5195         amdgpu_ring_write(ring, gds_base);
5196
5197         /* GDS Size */
5198         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5199         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5200                                 WRITE_DATA_DST_SEL(0)));
5201         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size);
5202         amdgpu_ring_write(ring, 0);
5203         amdgpu_ring_write(ring, gds_size);
5204
5205         /* GWS */
5206         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5207         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5208                                 WRITE_DATA_DST_SEL(0)));
5209         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws);
5210         amdgpu_ring_write(ring, 0);
5211         amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
5212
5213         /* OA */
5214         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5215         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5216                                 WRITE_DATA_DST_SEL(0)));
5217         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa);
5218         amdgpu_ring_write(ring, 0);
5219         amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
5220 }
5221
5222 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
5223 {
5224         WREG32(mmSQ_IND_INDEX,
5225                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5226                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5227                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
5228                 (SQ_IND_INDEX__FORCE_READ_MASK));
5229         return RREG32(mmSQ_IND_DATA);
5230 }
5231
5232 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
5233                            uint32_t wave, uint32_t thread,
5234                            uint32_t regno, uint32_t num, uint32_t *out)
5235 {
5236         WREG32(mmSQ_IND_INDEX,
5237                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5238                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5239                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
5240                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
5241                 (SQ_IND_INDEX__FORCE_READ_MASK) |
5242                 (SQ_IND_INDEX__AUTO_INCR_MASK));
5243         while (num--)
5244                 *(out++) = RREG32(mmSQ_IND_DATA);
5245 }
5246
5247 static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
5248 {
5249         /* type 0 wave data */
5250         dst[(*no_fields)++] = 0;
5251         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
5252         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
5253         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
5254         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
5255         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
5256         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
5257         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
5258         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
5259         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
5260         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
5261         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
5262         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
5263         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
5264         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
5265         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
5266         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
5267         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
5268         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
5269         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
5270 }
5271
5272 static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
5273                                      uint32_t wave, uint32_t start,
5274                                      uint32_t size, uint32_t *dst)
5275 {
5276         wave_read_regs(
5277                 adev, simd, wave, 0,
5278                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
5279 }
5280
5281
5282 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = {
5283         .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,
5284         .select_se_sh = &gfx_v8_0_select_se_sh,
5285         .read_wave_data = &gfx_v8_0_read_wave_data,
5286         .read_wave_sgprs = &gfx_v8_0_read_wave_sgprs,
5287         .select_me_pipe_q = &gfx_v8_0_select_me_pipe_q
5288 };
5289
5290 static int gfx_v8_0_early_init(void *handle)
5291 {
5292         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5293
5294         adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS;
5295         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
5296                                           AMDGPU_MAX_COMPUTE_RINGS);
5297         adev->gfx.funcs = &gfx_v8_0_gfx_funcs;
5298         gfx_v8_0_set_ring_funcs(adev);
5299         gfx_v8_0_set_irq_funcs(adev);
5300         gfx_v8_0_set_gds_init(adev);
5301         gfx_v8_0_set_rlc_funcs(adev);
5302
5303         return 0;
5304 }
5305
5306 static int gfx_v8_0_late_init(void *handle)
5307 {
5308         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5309         int r;
5310
5311         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5312         if (r)
5313                 return r;
5314
5315         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5316         if (r)
5317                 return r;
5318
5319         /* requires IBs so do in late init after IB pool is initialized */
5320         r = gfx_v8_0_do_edc_gpr_workarounds(adev);
5321         if (r)
5322                 return r;
5323
5324         r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
5325         if (r) {
5326                 DRM_ERROR("amdgpu_irq_get() failed to get IRQ for EDC, r: %d.\n", r);
5327                 return r;
5328         }
5329
5330         r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0);
5331         if (r) {
5332                 DRM_ERROR(
5333                         "amdgpu_irq_get() failed to get IRQ for SQ, r: %d.\n",
5334                         r);
5335                 return r;
5336         }
5337
5338         return 0;
5339 }
5340
5341 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
5342                                                        bool enable)
5343 {
5344         if ((adev->asic_type == CHIP_POLARIS11) ||
5345             (adev->asic_type == CHIP_POLARIS12) ||
5346             (adev->asic_type == CHIP_VEGAM))
5347                 /* Send msg to SMU via Powerplay */
5348                 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable);
5349
5350         WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0);
5351 }
5352
5353 static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
5354                                                         bool enable)
5355 {
5356         WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0);
5357 }
5358
5359 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev,
5360                 bool enable)
5361 {
5362         WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0);
5363 }
5364
5365 static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
5366                                           bool enable)
5367 {
5368         WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0);
5369 }
5370
5371 static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev,
5372                                                 bool enable)
5373 {
5374         WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0);
5375
5376         /* Read any GFX register to wake up GFX. */
5377         if (!enable)
5378                 RREG32(mmDB_RENDER_CONTROL);
5379 }
5380
5381 static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev,
5382                                           bool enable)
5383 {
5384         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
5385                 cz_enable_gfx_cg_power_gating(adev, true);
5386                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
5387                         cz_enable_gfx_pipeline_power_gating(adev, true);
5388         } else {
5389                 cz_enable_gfx_cg_power_gating(adev, false);
5390                 cz_enable_gfx_pipeline_power_gating(adev, false);
5391         }
5392 }
5393
5394 static int gfx_v8_0_set_powergating_state(void *handle,
5395                                           enum amd_powergating_state state)
5396 {
5397         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5398         bool enable = (state == AMD_PG_STATE_GATE);
5399
5400         if (amdgpu_sriov_vf(adev))
5401                 return 0;
5402
5403         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |
5404                                 AMD_PG_SUPPORT_RLC_SMU_HS |
5405                                 AMD_PG_SUPPORT_CP |
5406                                 AMD_PG_SUPPORT_GFX_DMG))
5407                 amdgpu_gfx_rlc_enter_safe_mode(adev);
5408         switch (adev->asic_type) {
5409         case CHIP_CARRIZO:
5410         case CHIP_STONEY:
5411
5412                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5413                         cz_enable_sck_slow_down_on_power_up(adev, true);
5414                         cz_enable_sck_slow_down_on_power_down(adev, true);
5415                 } else {
5416                         cz_enable_sck_slow_down_on_power_up(adev, false);
5417                         cz_enable_sck_slow_down_on_power_down(adev, false);
5418                 }
5419                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5420                         cz_enable_cp_power_gating(adev, true);
5421                 else
5422                         cz_enable_cp_power_gating(adev, false);
5423
5424                 cz_update_gfx_cg_power_gating(adev, enable);
5425
5426                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5427                         gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5428                 else
5429                         gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5430
5431                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5432                         gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5433                 else
5434                         gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5435                 break;
5436         case CHIP_POLARIS11:
5437         case CHIP_POLARIS12:
5438         case CHIP_VEGAM:
5439                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5440                         gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5441                 else
5442                         gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5443
5444                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5445                         gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5446                 else
5447                         gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5448
5449                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable)
5450                         polaris11_enable_gfx_quick_mg_power_gating(adev, true);
5451                 else
5452                         polaris11_enable_gfx_quick_mg_power_gating(adev, false);
5453                 break;
5454         default:
5455                 break;
5456         }
5457         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |
5458                                 AMD_PG_SUPPORT_RLC_SMU_HS |
5459                                 AMD_PG_SUPPORT_CP |
5460                                 AMD_PG_SUPPORT_GFX_DMG))
5461                 amdgpu_gfx_rlc_exit_safe_mode(adev);
5462         return 0;
5463 }
5464
5465 static void gfx_v8_0_get_clockgating_state(void *handle, u64 *flags)
5466 {
5467         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5468         int data;
5469
5470         if (amdgpu_sriov_vf(adev))
5471                 *flags = 0;
5472
5473         /* AMD_CG_SUPPORT_GFX_MGCG */
5474         data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5475         if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK))
5476                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5477
5478         /* AMD_CG_SUPPORT_GFX_CGLG */
5479         data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5480         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5481                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5482
5483         /* AMD_CG_SUPPORT_GFX_CGLS */
5484         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5485                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5486
5487         /* AMD_CG_SUPPORT_GFX_CGTS */
5488         data = RREG32(mmCGTS_SM_CTRL_REG);
5489         if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK))
5490                 *flags |= AMD_CG_SUPPORT_GFX_CGTS;
5491
5492         /* AMD_CG_SUPPORT_GFX_CGTS_LS */
5493         if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK))
5494                 *flags |= AMD_CG_SUPPORT_GFX_CGTS_LS;
5495
5496         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5497         data = RREG32(mmRLC_MEM_SLP_CNTL);
5498         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5499                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5500
5501         /* AMD_CG_SUPPORT_GFX_CP_LS */
5502         data = RREG32(mmCP_MEM_SLP_CNTL);
5503         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5504                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5505 }
5506
5507 static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
5508                                      uint32_t reg_addr, uint32_t cmd)
5509 {
5510         uint32_t data;
5511
5512         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
5513
5514         WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5515         WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5516
5517         data = RREG32(mmRLC_SERDES_WR_CTRL);
5518         if (adev->asic_type == CHIP_STONEY)
5519                 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5520                           RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5521                           RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5522                           RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5523                           RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5524                           RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5525                           RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5526                           RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5527                           RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
5528         else
5529                 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5530                           RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5531                           RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5532                           RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5533                           RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5534                           RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5535                           RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5536                           RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5537                           RLC_SERDES_WR_CTRL__BPM_DATA_MASK |
5538                           RLC_SERDES_WR_CTRL__REG_ADDR_MASK |
5539                           RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
5540         data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK |
5541                  (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) |
5542                  (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) |
5543                  (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT));
5544
5545         WREG32(mmRLC_SERDES_WR_CTRL, data);
5546 }
5547
5548 #define MSG_ENTER_RLC_SAFE_MODE     1
5549 #define MSG_EXIT_RLC_SAFE_MODE      0
5550 #define RLC_GPR_REG2__REQ_MASK 0x00000001
5551 #define RLC_GPR_REG2__REQ__SHIFT 0
5552 #define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001
5553 #define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e
5554
5555 static bool gfx_v8_0_is_rlc_enabled(struct amdgpu_device *adev)
5556 {
5557         uint32_t rlc_setting;
5558
5559         rlc_setting = RREG32(mmRLC_CNTL);
5560         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
5561                 return false;
5562
5563         return true;
5564 }
5565
5566 static void gfx_v8_0_set_safe_mode(struct amdgpu_device *adev)
5567 {
5568         uint32_t data;
5569         unsigned i;
5570         data = RREG32(mmRLC_CNTL);
5571         data |= RLC_SAFE_MODE__CMD_MASK;
5572         data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5573         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5574         WREG32(mmRLC_SAFE_MODE, data);
5575
5576         /* wait for RLC_SAFE_MODE */
5577         for (i = 0; i < adev->usec_timeout; i++) {
5578                 if ((RREG32(mmRLC_GPM_STAT) &
5579                      (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
5580                       RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) ==
5581                     (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
5582                      RLC_GPM_STAT__GFX_POWER_STATUS_MASK))
5583                         break;
5584                 udelay(1);
5585         }
5586         for (i = 0; i < adev->usec_timeout; i++) {
5587                 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
5588                         break;
5589                 udelay(1);
5590         }
5591 }
5592
5593 static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev)
5594 {
5595         uint32_t data;
5596         unsigned i;
5597
5598         data = RREG32(mmRLC_CNTL);
5599         data |= RLC_SAFE_MODE__CMD_MASK;
5600         data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5601         WREG32(mmRLC_SAFE_MODE, data);
5602
5603         for (i = 0; i < adev->usec_timeout; i++) {
5604                 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
5605                         break;
5606                 udelay(1);
5607         }
5608 }
5609
5610 static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
5611 {
5612         u32 data;
5613
5614         amdgpu_gfx_off_ctrl(adev, false);
5615
5616         if (amdgpu_sriov_is_pp_one_vf(adev))
5617                 data = RREG32_NO_KIQ(mmRLC_SPM_VMID);
5618         else
5619                 data = RREG32(mmRLC_SPM_VMID);
5620
5621         data &= ~RLC_SPM_VMID__RLC_SPM_VMID_MASK;
5622         data |= (vmid & RLC_SPM_VMID__RLC_SPM_VMID_MASK) << RLC_SPM_VMID__RLC_SPM_VMID__SHIFT;
5623
5624         if (amdgpu_sriov_is_pp_one_vf(adev))
5625                 WREG32_NO_KIQ(mmRLC_SPM_VMID, data);
5626         else
5627                 WREG32(mmRLC_SPM_VMID, data);
5628
5629         amdgpu_gfx_off_ctrl(adev, true);
5630 }
5631
5632 static const struct amdgpu_rlc_funcs iceland_rlc_funcs = {
5633         .is_rlc_enabled = gfx_v8_0_is_rlc_enabled,
5634         .set_safe_mode = gfx_v8_0_set_safe_mode,
5635         .unset_safe_mode = gfx_v8_0_unset_safe_mode,
5636         .init = gfx_v8_0_rlc_init,
5637         .get_csb_size = gfx_v8_0_get_csb_size,
5638         .get_csb_buffer = gfx_v8_0_get_csb_buffer,
5639         .get_cp_table_num = gfx_v8_0_cp_jump_table_num,
5640         .resume = gfx_v8_0_rlc_resume,
5641         .stop = gfx_v8_0_rlc_stop,
5642         .reset = gfx_v8_0_rlc_reset,
5643         .start = gfx_v8_0_rlc_start,
5644         .update_spm_vmid = gfx_v8_0_update_spm_vmid
5645 };
5646
5647 static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5648                                                       bool enable)
5649 {
5650         uint32_t temp, data;
5651
5652         amdgpu_gfx_rlc_enter_safe_mode(adev);
5653
5654         /* It is disabled by HW by default */
5655         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
5656                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5657                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS)
5658                                 /* 1 - RLC memory Light sleep */
5659                                 WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1);
5660
5661                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS)
5662                                 WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1);
5663                 }
5664
5665                 /* 3 - RLC_CGTT_MGCG_OVERRIDE */
5666                 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5667                 if (adev->flags & AMD_IS_APU)
5668                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5669                                   RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5670                                   RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK);
5671                 else
5672                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5673                                   RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5674                                   RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
5675                                   RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
5676
5677                 if (temp != data)
5678                         WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
5679
5680                 /* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5681                 gfx_v8_0_wait_for_rlc_serdes(adev);
5682
5683                 /* 5 - clear mgcg override */
5684                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
5685
5686                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) {
5687                         /* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */
5688                         temp = data = RREG32(mmCGTS_SM_CTRL_REG);
5689                         data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK);
5690                         data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT);
5691                         data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK;
5692                         data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK;
5693                         if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) &&
5694                             (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS))
5695                                 data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
5696                         data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK;
5697                         data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT);
5698                         if (temp != data)
5699                                 WREG32(mmCGTS_SM_CTRL_REG, data);
5700                 }
5701                 udelay(50);
5702
5703                 /* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5704                 gfx_v8_0_wait_for_rlc_serdes(adev);
5705         } else {
5706                 /* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */
5707                 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5708                 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5709                                 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5710                                 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
5711                                 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
5712                 if (temp != data)
5713                         WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
5714
5715                 /* 2 - disable MGLS in RLC */
5716                 data = RREG32(mmRLC_MEM_SLP_CNTL);
5717                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
5718                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
5719                         WREG32(mmRLC_MEM_SLP_CNTL, data);
5720                 }
5721
5722                 /* 3 - disable MGLS in CP */
5723                 data = RREG32(mmCP_MEM_SLP_CNTL);
5724                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
5725                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5726                         WREG32(mmCP_MEM_SLP_CNTL, data);
5727                 }
5728
5729                 /* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */
5730                 temp = data = RREG32(mmCGTS_SM_CTRL_REG);
5731                 data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK |
5732                                 CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK);
5733                 if (temp != data)
5734                         WREG32(mmCGTS_SM_CTRL_REG, data);
5735
5736                 /* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5737                 gfx_v8_0_wait_for_rlc_serdes(adev);
5738
5739                 /* 6 - set mgcg override */
5740                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD);
5741
5742                 udelay(50);
5743
5744                 /* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5745                 gfx_v8_0_wait_for_rlc_serdes(adev);
5746         }
5747
5748         amdgpu_gfx_rlc_exit_safe_mode(adev);
5749 }
5750
5751 static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5752                                                       bool enable)
5753 {
5754         uint32_t temp, temp1, data, data1;
5755
5756         temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5757
5758         amdgpu_gfx_rlc_enter_safe_mode(adev);
5759
5760         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5761                 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5762                 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK;
5763                 if (temp1 != data1)
5764                         WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5765
5766                 /* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5767                 gfx_v8_0_wait_for_rlc_serdes(adev);
5768
5769                 /* 2 - clear cgcg override */
5770                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
5771
5772                 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5773                 gfx_v8_0_wait_for_rlc_serdes(adev);
5774
5775                 /* 3 - write cmd to set CGLS */
5776                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD);
5777
5778                 /* 4 - enable cgcg */
5779                 data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5780
5781                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5782                         /* enable cgls*/
5783                         data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5784
5785                         temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5786                         data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK;
5787
5788                         if (temp1 != data1)
5789                                 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5790                 } else {
5791                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5792                 }
5793
5794                 if (temp != data)
5795                         WREG32(mmRLC_CGCG_CGLS_CTRL, data);
5796
5797                 /* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/
5798                  * Cmp_busy/GFX_Idle interrupts
5799                  */
5800                 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5801         } else {
5802                 /* disable cntx_empty_int_enable & GFX Idle interrupt */
5803                 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
5804
5805                 /* TEST CGCG */
5806                 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5807                 data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK |
5808                                 RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK);
5809                 if (temp1 != data1)
5810                         WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5811
5812                 /* read gfx register to wake up cgcg */
5813                 RREG32(mmCB_CGTT_SCLK_CTRL);
5814                 RREG32(mmCB_CGTT_SCLK_CTRL);
5815                 RREG32(mmCB_CGTT_SCLK_CTRL);
5816                 RREG32(mmCB_CGTT_SCLK_CTRL);
5817
5818                 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5819                 gfx_v8_0_wait_for_rlc_serdes(adev);
5820
5821                 /* write cmd to Set CGCG Override */
5822                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD);
5823
5824                 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5825                 gfx_v8_0_wait_for_rlc_serdes(adev);
5826
5827                 /* write cmd to Clear CGLS */
5828                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD);
5829
5830                 /* disable cgcg, cgls should be disabled too. */
5831                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK |
5832                           RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5833                 if (temp != data)
5834                         WREG32(mmRLC_CGCG_CGLS_CTRL, data);
5835                 /* enable interrupts again for PG */
5836                 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5837         }
5838
5839         gfx_v8_0_wait_for_rlc_serdes(adev);
5840
5841         amdgpu_gfx_rlc_exit_safe_mode(adev);
5842 }
5843 static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5844                                             bool enable)
5845 {
5846         if (enable) {
5847                 /* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS)
5848                  * ===  MGCG + MGLS + TS(CG/LS) ===
5849                  */
5850                 gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
5851                 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
5852         } else {
5853                 /* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS)
5854                  * ===  CGCG + CGLS ===
5855                  */
5856                 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
5857                 gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
5858         }
5859         return 0;
5860 }
5861
5862 static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
5863                                           enum amd_clockgating_state state)
5864 {
5865         uint32_t msg_id, pp_state = 0;
5866         uint32_t pp_support_state = 0;
5867
5868         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
5869                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5870                         pp_support_state = PP_STATE_SUPPORT_LS;
5871                         pp_state = PP_STATE_LS;
5872                 }
5873                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5874                         pp_support_state |= PP_STATE_SUPPORT_CG;
5875                         pp_state |= PP_STATE_CG;
5876                 }
5877                 if (state == AMD_CG_STATE_UNGATE)
5878                         pp_state = 0;
5879
5880                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5881                                 PP_BLOCK_GFX_CG,
5882                                 pp_support_state,
5883                                 pp_state);
5884                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5885         }
5886
5887         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
5888                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5889                         pp_support_state = PP_STATE_SUPPORT_LS;
5890                         pp_state = PP_STATE_LS;
5891                 }
5892
5893                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5894                         pp_support_state |= PP_STATE_SUPPORT_CG;
5895                         pp_state |= PP_STATE_CG;
5896                 }
5897
5898                 if (state == AMD_CG_STATE_UNGATE)
5899                         pp_state = 0;
5900
5901                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5902                                 PP_BLOCK_GFX_MG,
5903                                 pp_support_state,
5904                                 pp_state);
5905                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5906         }
5907
5908         return 0;
5909 }
5910
5911 static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
5912                                           enum amd_clockgating_state state)
5913 {
5914
5915         uint32_t msg_id, pp_state = 0;
5916         uint32_t pp_support_state = 0;
5917
5918         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
5919                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5920                         pp_support_state = PP_STATE_SUPPORT_LS;
5921                         pp_state = PP_STATE_LS;
5922                 }
5923                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5924                         pp_support_state |= PP_STATE_SUPPORT_CG;
5925                         pp_state |= PP_STATE_CG;
5926                 }
5927                 if (state == AMD_CG_STATE_UNGATE)
5928                         pp_state = 0;
5929
5930                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5931                                 PP_BLOCK_GFX_CG,
5932                                 pp_support_state,
5933                                 pp_state);
5934                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5935         }
5936
5937         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) {
5938                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5939                         pp_support_state = PP_STATE_SUPPORT_LS;
5940                         pp_state = PP_STATE_LS;
5941                 }
5942                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5943                         pp_support_state |= PP_STATE_SUPPORT_CG;
5944                         pp_state |= PP_STATE_CG;
5945                 }
5946                 if (state == AMD_CG_STATE_UNGATE)
5947                         pp_state = 0;
5948
5949                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5950                                 PP_BLOCK_GFX_3D,
5951                                 pp_support_state,
5952                                 pp_state);
5953                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5954         }
5955
5956         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
5957                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5958                         pp_support_state = PP_STATE_SUPPORT_LS;
5959                         pp_state = PP_STATE_LS;
5960                 }
5961
5962                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5963                         pp_support_state |= PP_STATE_SUPPORT_CG;
5964                         pp_state |= PP_STATE_CG;
5965                 }
5966
5967                 if (state == AMD_CG_STATE_UNGATE)
5968                         pp_state = 0;
5969
5970                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5971                                 PP_BLOCK_GFX_MG,
5972                                 pp_support_state,
5973                                 pp_state);
5974                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5975         }
5976
5977         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
5978                 pp_support_state = PP_STATE_SUPPORT_LS;
5979
5980                 if (state == AMD_CG_STATE_UNGATE)
5981                         pp_state = 0;
5982                 else
5983                         pp_state = PP_STATE_LS;
5984
5985                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5986                                 PP_BLOCK_GFX_RLC,
5987                                 pp_support_state,
5988                                 pp_state);
5989                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5990         }
5991
5992         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
5993                 pp_support_state = PP_STATE_SUPPORT_LS;
5994
5995                 if (state == AMD_CG_STATE_UNGATE)
5996                         pp_state = 0;
5997                 else
5998                         pp_state = PP_STATE_LS;
5999                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
6000                         PP_BLOCK_GFX_CP,
6001                         pp_support_state,
6002                         pp_state);
6003                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
6004         }
6005
6006         return 0;
6007 }
6008
6009 static int gfx_v8_0_set_clockgating_state(void *handle,
6010                                           enum amd_clockgating_state state)
6011 {
6012         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6013
6014         if (amdgpu_sriov_vf(adev))
6015                 return 0;
6016
6017         switch (adev->asic_type) {
6018         case CHIP_FIJI:
6019         case CHIP_CARRIZO:
6020         case CHIP_STONEY:
6021                 gfx_v8_0_update_gfx_clock_gating(adev,
6022                                                  state == AMD_CG_STATE_GATE);
6023                 break;
6024         case CHIP_TONGA:
6025                 gfx_v8_0_tonga_update_gfx_clock_gating(adev, state);
6026                 break;
6027         case CHIP_POLARIS10:
6028         case CHIP_POLARIS11:
6029         case CHIP_POLARIS12:
6030         case CHIP_VEGAM:
6031                 gfx_v8_0_polaris_update_gfx_clock_gating(adev, state);
6032                 break;
6033         default:
6034                 break;
6035         }
6036         return 0;
6037 }
6038
6039 static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring)
6040 {
6041         return *ring->rptr_cpu_addr;
6042 }
6043
6044 static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
6045 {
6046         struct amdgpu_device *adev = ring->adev;
6047
6048         if (ring->use_doorbell)
6049                 /* XXX check if swapping is necessary on BE */
6050                 return *ring->wptr_cpu_addr;
6051         else
6052                 return RREG32(mmCP_RB0_WPTR);
6053 }
6054
6055 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
6056 {
6057         struct amdgpu_device *adev = ring->adev;
6058
6059         if (ring->use_doorbell) {
6060                 /* XXX check if swapping is necessary on BE */
6061                 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
6062                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
6063         } else {
6064                 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
6065                 (void)RREG32(mmCP_RB0_WPTR);
6066         }
6067 }
6068
6069 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
6070 {
6071         u32 ref_and_mask, reg_mem_engine;
6072
6073         if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) ||
6074             (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) {
6075                 switch (ring->me) {
6076                 case 1:
6077                         ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
6078                         break;
6079                 case 2:
6080                         ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
6081                         break;
6082                 default:
6083                         return;
6084                 }
6085                 reg_mem_engine = 0;
6086         } else {
6087                 ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
6088                 reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */
6089         }
6090
6091         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6092         amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
6093                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
6094                                  reg_mem_engine));
6095         amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ);
6096         amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE);
6097         amdgpu_ring_write(ring, ref_and_mask);
6098         amdgpu_ring_write(ring, ref_and_mask);
6099         amdgpu_ring_write(ring, 0x20); /* poll interval */
6100 }
6101
6102 static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
6103 {
6104         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6105         amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
6106                 EVENT_INDEX(4));
6107
6108         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6109         amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
6110                 EVENT_INDEX(0));
6111 }
6112
6113 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
6114                                         struct amdgpu_job *job,
6115                                         struct amdgpu_ib *ib,
6116                                         uint32_t flags)
6117 {
6118         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
6119         u32 header, control = 0;
6120
6121         if (ib->flags & AMDGPU_IB_FLAG_CE)
6122                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
6123         else
6124                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
6125
6126         control |= ib->length_dw | (vmid << 24);
6127
6128         if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
6129                 control |= INDIRECT_BUFFER_PRE_ENB(1);
6130
6131                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
6132                         gfx_v8_0_ring_emit_de_meta(ring);
6133         }
6134
6135         amdgpu_ring_write(ring, header);
6136         amdgpu_ring_write(ring,
6137 #ifdef __BIG_ENDIAN
6138                           (2 << 0) |
6139 #endif
6140                           (ib->gpu_addr & 0xFFFFFFFC));
6141         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6142         amdgpu_ring_write(ring, control);
6143 }
6144
6145 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
6146                                           struct amdgpu_job *job,
6147                                           struct amdgpu_ib *ib,
6148                                           uint32_t flags)
6149 {
6150         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
6151         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
6152
6153         /* Currently, there is a high possibility to get wave ID mismatch
6154          * between ME and GDS, leading to a hw deadlock, because ME generates
6155          * different wave IDs than the GDS expects. This situation happens
6156          * randomly when at least 5 compute pipes use GDS ordered append.
6157          * The wave IDs generated by ME are also wrong after suspend/resume.
6158          * Those are probably bugs somewhere else in the kernel driver.
6159          *
6160          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
6161          * GDS to 0 for this ring (me/pipe).
6162          */
6163         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
6164                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
6165                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START);
6166                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
6167         }
6168
6169         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
6170         amdgpu_ring_write(ring,
6171 #ifdef __BIG_ENDIAN
6172                                 (2 << 0) |
6173 #endif
6174                                 (ib->gpu_addr & 0xFFFFFFFC));
6175         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6176         amdgpu_ring_write(ring, control);
6177 }
6178
6179 static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
6180                                          u64 seq, unsigned flags)
6181 {
6182         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6183         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6184
6185         /* Workaround for cache flush problems. First send a dummy EOP
6186          * event down the pipe with seq one below.
6187          */
6188         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
6189         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6190                                  EOP_TC_ACTION_EN |
6191                                  EOP_TC_WB_ACTION_EN |
6192                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6193                                  EVENT_INDEX(5)));
6194         amdgpu_ring_write(ring, addr & 0xfffffffc);
6195         amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
6196                                 DATA_SEL(1) | INT_SEL(0));
6197         amdgpu_ring_write(ring, lower_32_bits(seq - 1));
6198         amdgpu_ring_write(ring, upper_32_bits(seq - 1));
6199
6200         /* Then send the real EOP event down the pipe:
6201          * EVENT_WRITE_EOP - flush caches, send int */
6202         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
6203         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6204                                  EOP_TC_ACTION_EN |
6205                                  EOP_TC_WB_ACTION_EN |
6206                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6207                                  EVENT_INDEX(5)));
6208         amdgpu_ring_write(ring, addr & 0xfffffffc);
6209         amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
6210                           DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
6211         amdgpu_ring_write(ring, lower_32_bits(seq));
6212         amdgpu_ring_write(ring, upper_32_bits(seq));
6213
6214 }
6215
6216 static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
6217 {
6218         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6219         uint32_t seq = ring->fence_drv.sync_seq;
6220         uint64_t addr = ring->fence_drv.gpu_addr;
6221
6222         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6223         amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
6224                                  WAIT_REG_MEM_FUNCTION(3) | /* equal */
6225                                  WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */
6226         amdgpu_ring_write(ring, addr & 0xfffffffc);
6227         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
6228         amdgpu_ring_write(ring, seq);
6229         amdgpu_ring_write(ring, 0xffffffff);
6230         amdgpu_ring_write(ring, 4); /* poll interval */
6231 }
6232
6233 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
6234                                         unsigned vmid, uint64_t pd_addr)
6235 {
6236         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6237
6238         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
6239
6240         /* wait for the invalidate to complete */
6241         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6242         amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6243                                  WAIT_REG_MEM_FUNCTION(0) |  /* always */
6244                                  WAIT_REG_MEM_ENGINE(0))); /* me */
6245         amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
6246         amdgpu_ring_write(ring, 0);
6247         amdgpu_ring_write(ring, 0); /* ref */
6248         amdgpu_ring_write(ring, 0); /* mask */
6249         amdgpu_ring_write(ring, 0x20); /* poll interval */
6250
6251         /* compute doesn't have PFP */
6252         if (usepfp) {
6253                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6254                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6255                 amdgpu_ring_write(ring, 0x0);
6256         }
6257 }
6258
6259 static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
6260 {
6261         return *ring->wptr_cpu_addr;
6262 }
6263
6264 static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
6265 {
6266         struct amdgpu_device *adev = ring->adev;
6267
6268         /* XXX check if swapping is necessary on BE */
6269         *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
6270         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
6271 }
6272
6273 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
6274                                              u64 addr, u64 seq,
6275                                              unsigned flags)
6276 {
6277         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6278         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6279
6280         /* RELEASE_MEM - flush caches, send int */
6281         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
6282         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6283                                  EOP_TC_ACTION_EN |
6284                                  EOP_TC_WB_ACTION_EN |
6285                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6286                                  EVENT_INDEX(5)));
6287         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
6288         amdgpu_ring_write(ring, addr & 0xfffffffc);
6289         amdgpu_ring_write(ring, upper_32_bits(addr));
6290         amdgpu_ring_write(ring, lower_32_bits(seq));
6291         amdgpu_ring_write(ring, upper_32_bits(seq));
6292 }
6293
6294 static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
6295                                          u64 seq, unsigned int flags)
6296 {
6297         /* we only allocate 32bit for each seq wb address */
6298         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
6299
6300         /* write fence seq to the "addr" */
6301         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6302         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6303                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
6304         amdgpu_ring_write(ring, lower_32_bits(addr));
6305         amdgpu_ring_write(ring, upper_32_bits(addr));
6306         amdgpu_ring_write(ring, lower_32_bits(seq));
6307
6308         if (flags & AMDGPU_FENCE_FLAG_INT) {
6309                 /* set register to trigger INT */
6310                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6311                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6312                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
6313                 amdgpu_ring_write(ring, mmCPC_INT_STATUS);
6314                 amdgpu_ring_write(ring, 0);
6315                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
6316         }
6317 }
6318
6319 static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
6320 {
6321         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
6322         amdgpu_ring_write(ring, 0);
6323 }
6324
6325 static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
6326 {
6327         uint32_t dw2 = 0;
6328
6329         if (amdgpu_sriov_vf(ring->adev))
6330                 gfx_v8_0_ring_emit_ce_meta(ring);
6331
6332         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
6333         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
6334                 gfx_v8_0_ring_emit_vgt_flush(ring);
6335                 /* set load_global_config & load_global_uconfig */
6336                 dw2 |= 0x8001;
6337                 /* set load_cs_sh_regs */
6338                 dw2 |= 0x01000000;
6339                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
6340                 dw2 |= 0x10002;
6341
6342                 /* set load_ce_ram if preamble presented */
6343                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
6344                         dw2 |= 0x10000000;
6345         } else {
6346                 /* still load_ce_ram if this is the first time preamble presented
6347                  * although there is no context switch happens.
6348                  */
6349                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
6350                         dw2 |= 0x10000000;
6351         }
6352
6353         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6354         amdgpu_ring_write(ring, dw2);
6355         amdgpu_ring_write(ring, 0);
6356 }
6357
6358 static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
6359 {
6360         unsigned ret;
6361
6362         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
6363         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
6364         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
6365         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
6366         ret = ring->wptr & ring->buf_mask;
6367         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
6368         return ret;
6369 }
6370
6371 static void gfx_v8_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
6372 {
6373         unsigned cur;
6374
6375         BUG_ON(offset > ring->buf_mask);
6376         BUG_ON(ring->ring[offset] != 0x55aa55aa);
6377
6378         cur = (ring->wptr & ring->buf_mask) - 1;
6379         if (likely(cur > offset))
6380                 ring->ring[offset] = cur - offset;
6381         else
6382                 ring->ring[offset] = (ring->ring_size >> 2) - offset + cur;
6383 }
6384
6385 static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
6386                                     uint32_t reg_val_offs)
6387 {
6388         struct amdgpu_device *adev = ring->adev;
6389
6390         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6391         amdgpu_ring_write(ring, 0 |     /* src: register*/
6392                                 (5 << 8) |      /* dst: memory */
6393                                 (1 << 20));     /* write confirm */
6394         amdgpu_ring_write(ring, reg);
6395         amdgpu_ring_write(ring, 0);
6396         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6397                                 reg_val_offs * 4));
6398         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6399                                 reg_val_offs * 4));
6400 }
6401
6402 static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6403                                   uint32_t val)
6404 {
6405         uint32_t cmd;
6406
6407         switch (ring->funcs->type) {
6408         case AMDGPU_RING_TYPE_GFX:
6409                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
6410                 break;
6411         case AMDGPU_RING_TYPE_KIQ:
6412                 cmd = 1 << 16; /* no inc addr */
6413                 break;
6414         default:
6415                 cmd = WR_CONFIRM;
6416                 break;
6417         }
6418
6419         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6420         amdgpu_ring_write(ring, cmd);
6421         amdgpu_ring_write(ring, reg);
6422         amdgpu_ring_write(ring, 0);
6423         amdgpu_ring_write(ring, val);
6424 }
6425
6426 static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
6427 {
6428         struct amdgpu_device *adev = ring->adev;
6429         uint32_t value = 0;
6430
6431         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
6432         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
6433         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
6434         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
6435         WREG32(mmSQ_CMD, value);
6436 }
6437
6438 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6439                                                  enum amdgpu_interrupt_state state)
6440 {
6441         WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE,
6442                      state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6443 }
6444
6445 static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6446                                                      int me, int pipe,
6447                                                      enum amdgpu_interrupt_state state)
6448 {
6449         u32 mec_int_cntl, mec_int_cntl_reg;
6450
6451         /*
6452          * amdgpu controls only the first MEC. That's why this function only
6453          * handles the setting of interrupts for this specific MEC. All other
6454          * pipes' interrupts are set by amdkfd.
6455          */
6456
6457         if (me == 1) {
6458                 switch (pipe) {
6459                 case 0:
6460                         mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL;
6461                         break;
6462                 case 1:
6463                         mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL;
6464                         break;
6465                 case 2:
6466                         mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL;
6467                         break;
6468                 case 3:
6469                         mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL;
6470                         break;
6471                 default:
6472                         DRM_DEBUG("invalid pipe %d\n", pipe);
6473                         return;
6474                 }
6475         } else {
6476                 DRM_DEBUG("invalid me %d\n", me);
6477                 return;
6478         }
6479
6480         switch (state) {
6481         case AMDGPU_IRQ_STATE_DISABLE:
6482                 mec_int_cntl = RREG32(mec_int_cntl_reg);
6483                 mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
6484                 WREG32(mec_int_cntl_reg, mec_int_cntl);
6485                 break;
6486         case AMDGPU_IRQ_STATE_ENABLE:
6487                 mec_int_cntl = RREG32(mec_int_cntl_reg);
6488                 mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
6489                 WREG32(mec_int_cntl_reg, mec_int_cntl);
6490                 break;
6491         default:
6492                 break;
6493         }
6494 }
6495
6496 static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6497                                              struct amdgpu_irq_src *source,
6498                                              unsigned type,
6499                                              enum amdgpu_interrupt_state state)
6500 {
6501         WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE,
6502                      state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6503
6504         return 0;
6505 }
6506
6507 static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6508                                               struct amdgpu_irq_src *source,
6509                                               unsigned type,
6510                                               enum amdgpu_interrupt_state state)
6511 {
6512         WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE,
6513                      state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6514
6515         return 0;
6516 }
6517
6518 static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6519                                             struct amdgpu_irq_src *src,
6520                                             unsigned type,
6521                                             enum amdgpu_interrupt_state state)
6522 {
6523         switch (type) {
6524         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6525                 gfx_v8_0_set_gfx_eop_interrupt_state(adev, state);
6526                 break;
6527         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6528                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6529                 break;
6530         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6531                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6532                 break;
6533         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6534                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6535                 break;
6536         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6537                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6538                 break;
6539         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6540                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6541                 break;
6542         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6543                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6544                 break;
6545         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6546                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6547                 break;
6548         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6549                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6550                 break;
6551         default:
6552                 break;
6553         }
6554         return 0;
6555 }
6556
6557 static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev,
6558                                          struct amdgpu_irq_src *source,
6559                                          unsigned int type,
6560                                          enum amdgpu_interrupt_state state)
6561 {
6562         int enable_flag;
6563
6564         switch (state) {
6565         case AMDGPU_IRQ_STATE_DISABLE:
6566                 enable_flag = 0;
6567                 break;
6568
6569         case AMDGPU_IRQ_STATE_ENABLE:
6570                 enable_flag = 1;
6571                 break;
6572
6573         default:
6574                 return -EINVAL;
6575         }
6576
6577         WREG32_FIELD(CP_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6578         WREG32_FIELD(CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6579         WREG32_FIELD(CP_INT_CNTL_RING1, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6580         WREG32_FIELD(CP_INT_CNTL_RING2, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6581         WREG32_FIELD(CPC_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6582         WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6583                      enable_flag);
6584         WREG32_FIELD(CP_ME1_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6585                      enable_flag);
6586         WREG32_FIELD(CP_ME1_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6587                      enable_flag);
6588         WREG32_FIELD(CP_ME1_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6589                      enable_flag);
6590         WREG32_FIELD(CP_ME2_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6591                      enable_flag);
6592         WREG32_FIELD(CP_ME2_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6593                      enable_flag);
6594         WREG32_FIELD(CP_ME2_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6595                      enable_flag);
6596         WREG32_FIELD(CP_ME2_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6597                      enable_flag);
6598
6599         return 0;
6600 }
6601
6602 static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev,
6603                                      struct amdgpu_irq_src *source,
6604                                      unsigned int type,
6605                                      enum amdgpu_interrupt_state state)
6606 {
6607         int enable_flag;
6608
6609         switch (state) {
6610         case AMDGPU_IRQ_STATE_DISABLE:
6611                 enable_flag = 1;
6612                 break;
6613
6614         case AMDGPU_IRQ_STATE_ENABLE:
6615                 enable_flag = 0;
6616                 break;
6617
6618         default:
6619                 return -EINVAL;
6620         }
6621
6622         WREG32_FIELD(SQ_INTERRUPT_MSG_CTRL, STALL,
6623                      enable_flag);
6624
6625         return 0;
6626 }
6627
6628 static int gfx_v8_0_eop_irq(struct amdgpu_device *adev,
6629                             struct amdgpu_irq_src *source,
6630                             struct amdgpu_iv_entry *entry)
6631 {
6632         int i;
6633         u8 me_id, pipe_id, queue_id;
6634         struct amdgpu_ring *ring;
6635
6636         DRM_DEBUG("IH: CP EOP\n");
6637         me_id = (entry->ring_id & 0x0c) >> 2;
6638         pipe_id = (entry->ring_id & 0x03) >> 0;
6639         queue_id = (entry->ring_id & 0x70) >> 4;
6640
6641         switch (me_id) {
6642         case 0:
6643                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6644                 break;
6645         case 1:
6646         case 2:
6647                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6648                         ring = &adev->gfx.compute_ring[i];
6649                         /* Per-queue interrupt is supported for MEC starting from VI.
6650                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
6651                           */
6652                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6653                                 amdgpu_fence_process(ring);
6654                 }
6655                 break;
6656         }
6657         return 0;
6658 }
6659
6660 static void gfx_v8_0_fault(struct amdgpu_device *adev,
6661                            struct amdgpu_iv_entry *entry)
6662 {
6663         u8 me_id, pipe_id, queue_id;
6664         struct amdgpu_ring *ring;
6665         int i;
6666
6667         me_id = (entry->ring_id & 0x0c) >> 2;
6668         pipe_id = (entry->ring_id & 0x03) >> 0;
6669         queue_id = (entry->ring_id & 0x70) >> 4;
6670
6671         switch (me_id) {
6672         case 0:
6673                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
6674                 break;
6675         case 1:
6676         case 2:
6677                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6678                         ring = &adev->gfx.compute_ring[i];
6679                         if (ring->me == me_id && ring->pipe == pipe_id &&
6680                             ring->queue == queue_id)
6681                                 drm_sched_fault(&ring->sched);
6682                 }
6683                 break;
6684         }
6685 }
6686
6687 static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev,
6688                                  struct amdgpu_irq_src *source,
6689                                  struct amdgpu_iv_entry *entry)
6690 {
6691         DRM_ERROR("Illegal register access in command stream\n");
6692         gfx_v8_0_fault(adev, entry);
6693         return 0;
6694 }
6695
6696 static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,
6697                                   struct amdgpu_irq_src *source,
6698                                   struct amdgpu_iv_entry *entry)
6699 {
6700         DRM_ERROR("Illegal instruction in command stream\n");
6701         gfx_v8_0_fault(adev, entry);
6702         return 0;
6703 }
6704
6705 static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev,
6706                                      struct amdgpu_irq_src *source,
6707                                      struct amdgpu_iv_entry *entry)
6708 {
6709         DRM_ERROR("CP EDC/ECC error detected.");
6710         return 0;
6711 }
6712
6713 static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data,
6714                                   bool from_wq)
6715 {
6716         u32 enc, se_id, sh_id, cu_id;
6717         char type[20];
6718         int sq_edc_source = -1;
6719
6720         enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING);
6721         se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID);
6722
6723         switch (enc) {
6724                 case 0:
6725                         DRM_INFO("SQ general purpose intr detected:"
6726                                         "se_id %d, immed_overflow %d, host_reg_overflow %d,"
6727                                         "host_cmd_overflow %d, cmd_timestamp %d,"
6728                                         "reg_timestamp %d, thread_trace_buff_full %d,"
6729                                         "wlt %d, thread_trace %d.\n",
6730                                         se_id,
6731                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW),
6732                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW),
6733                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW),
6734                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP),
6735                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP),
6736                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL),
6737                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT),
6738                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE)
6739                                         );
6740                         break;
6741                 case 1:
6742                 case 2:
6743
6744                         cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID);
6745                         sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID);
6746
6747                         /*
6748                          * This function can be called either directly from ISR
6749                          * or from BH in which case we can access SQ_EDC_INFO
6750                          * instance
6751                          */
6752                         if (from_wq) {
6753                                 mutex_lock(&adev->grbm_idx_mutex);
6754                                 gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id);
6755
6756                                 sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE);
6757
6758                                 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6759                                 mutex_unlock(&adev->grbm_idx_mutex);
6760                         }
6761
6762                         if (enc == 1)
6763                                 sprintf(type, "instruction intr");
6764                         else
6765                                 sprintf(type, "EDC/ECC error");
6766
6767                         DRM_INFO(
6768                                 "SQ %s detected: "
6769                                         "se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d "
6770                                         "trap %s, sq_ed_info.source %s.\n",
6771                                         type, se_id, sh_id, cu_id,
6772                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID),
6773                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID),
6774                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID),
6775                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false",
6776                                         (sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable"
6777                                 );
6778                         break;
6779                 default:
6780                         DRM_ERROR("SQ invalid encoding type\n.");
6781         }
6782 }
6783
6784 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work)
6785 {
6786
6787         struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work);
6788         struct sq_work *sq_work = container_of(work, struct sq_work, work);
6789
6790         gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data, true);
6791 }
6792
6793 static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,
6794                            struct amdgpu_irq_src *source,
6795                            struct amdgpu_iv_entry *entry)
6796 {
6797         unsigned ih_data = entry->src_data[0];
6798
6799         /*
6800          * Try to submit work so SQ_EDC_INFO can be accessed from
6801          * BH. If previous work submission hasn't finished yet
6802          * just print whatever info is possible directly from the ISR.
6803          */
6804         if (work_pending(&adev->gfx.sq_work.work)) {
6805                 gfx_v8_0_parse_sq_irq(adev, ih_data, false);
6806         } else {
6807                 adev->gfx.sq_work.ih_data = ih_data;
6808                 schedule_work(&adev->gfx.sq_work.work);
6809         }
6810
6811         return 0;
6812 }
6813
6814 static void gfx_v8_0_emit_mem_sync(struct amdgpu_ring *ring)
6815 {
6816         amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
6817         amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
6818                           PACKET3_TC_ACTION_ENA |
6819                           PACKET3_SH_KCACHE_ACTION_ENA |
6820                           PACKET3_SH_ICACHE_ACTION_ENA |
6821                           PACKET3_TC_WB_ACTION_ENA);  /* CP_COHER_CNTL */
6822         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6823         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE */
6824         amdgpu_ring_write(ring, 0x0000000A); /* poll interval */
6825 }
6826
6827 static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring)
6828 {
6829         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6830         amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
6831                           PACKET3_TC_ACTION_ENA |
6832                           PACKET3_SH_KCACHE_ACTION_ENA |
6833                           PACKET3_SH_ICACHE_ACTION_ENA |
6834                           PACKET3_TC_WB_ACTION_ENA);  /* CP_COHER_CNTL */
6835         amdgpu_ring_write(ring, 0xffffffff);    /* CP_COHER_SIZE */
6836         amdgpu_ring_write(ring, 0xff);          /* CP_COHER_SIZE_HI */
6837         amdgpu_ring_write(ring, 0);             /* CP_COHER_BASE */
6838         amdgpu_ring_write(ring, 0);             /* CP_COHER_BASE_HI */
6839         amdgpu_ring_write(ring, 0x0000000A);    /* poll interval */
6840 }
6841
6842
6843 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6844 #define mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT       0x0000007f
6845 static void gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6846                                         uint32_t pipe, bool enable)
6847 {
6848         uint32_t val;
6849         uint32_t wcl_cs_reg;
6850
6851         val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT;
6852
6853         switch (pipe) {
6854         case 0:
6855                 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS0;
6856                 break;
6857         case 1:
6858                 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS1;
6859                 break;
6860         case 2:
6861                 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS2;
6862                 break;
6863         case 3:
6864                 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS3;
6865                 break;
6866         default:
6867                 DRM_DEBUG("invalid pipe %d\n", pipe);
6868                 return;
6869         }
6870
6871         amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6872
6873 }
6874
6875 #define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT      0x07ffffff
6876 static void gfx_v8_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6877 {
6878         struct amdgpu_device *adev = ring->adev;
6879         uint32_t val;
6880         int i;
6881
6882         /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6883          * number of gfx waves. Setting 5 bit will make sure gfx only gets
6884          * around 25% of gpu resources.
6885          */
6886         val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6887         amdgpu_ring_emit_wreg(ring, mmSPI_WCL_PIPE_PERCENT_GFX, val);
6888
6889         /* Restrict waves for normal/low priority compute queues as well
6890          * to get best QoS for high priority compute jobs.
6891          *
6892          * amdgpu controls only 1st ME(0-3 CS pipes).
6893          */
6894         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6895                 if (i != ring->pipe)
6896                         gfx_v8_0_emit_wave_limit_cs(ring, i, enable);
6897
6898         }
6899
6900 }
6901
6902 static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
6903         .name = "gfx_v8_0",
6904         .early_init = gfx_v8_0_early_init,
6905         .late_init = gfx_v8_0_late_init,
6906         .sw_init = gfx_v8_0_sw_init,
6907         .sw_fini = gfx_v8_0_sw_fini,
6908         .hw_init = gfx_v8_0_hw_init,
6909         .hw_fini = gfx_v8_0_hw_fini,
6910         .suspend = gfx_v8_0_suspend,
6911         .resume = gfx_v8_0_resume,
6912         .is_idle = gfx_v8_0_is_idle,
6913         .wait_for_idle = gfx_v8_0_wait_for_idle,
6914         .check_soft_reset = gfx_v8_0_check_soft_reset,
6915         .pre_soft_reset = gfx_v8_0_pre_soft_reset,
6916         .soft_reset = gfx_v8_0_soft_reset,
6917         .post_soft_reset = gfx_v8_0_post_soft_reset,
6918         .set_clockgating_state = gfx_v8_0_set_clockgating_state,
6919         .set_powergating_state = gfx_v8_0_set_powergating_state,
6920         .get_clockgating_state = gfx_v8_0_get_clockgating_state,
6921 };
6922
6923 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
6924         .type = AMDGPU_RING_TYPE_GFX,
6925         .align_mask = 0xff,
6926         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6927         .support_64bit_ptrs = false,
6928         .get_rptr = gfx_v8_0_ring_get_rptr,
6929         .get_wptr = gfx_v8_0_ring_get_wptr_gfx,
6930         .set_wptr = gfx_v8_0_ring_set_wptr_gfx,
6931         .emit_frame_size = /* maximum 215dw if count 16 IBs in */
6932                 5 +  /* COND_EXEC */
6933                 7 +  /* PIPELINE_SYNC */
6934                 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */
6935                 12 +  /* FENCE for VM_FLUSH */
6936                 20 + /* GDS switch */
6937                 4 + /* double SWITCH_BUFFER,
6938                        the first COND_EXEC jump to the place just
6939                            prior to this double SWITCH_BUFFER  */
6940                 5 + /* COND_EXEC */
6941                 7 +      /*     HDP_flush */
6942                 4 +      /*     VGT_flush */
6943                 14 + /* CE_META */
6944                 31 + /* DE_META */
6945                 3 + /* CNTX_CTRL */
6946                 5 + /* HDP_INVL */
6947                 12 + 12 + /* FENCE x2 */
6948                 2 + /* SWITCH_BUFFER */
6949                 5, /* SURFACE_SYNC */
6950         .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_gfx */
6951         .emit_ib = gfx_v8_0_ring_emit_ib_gfx,
6952         .emit_fence = gfx_v8_0_ring_emit_fence_gfx,
6953         .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
6954         .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
6955         .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
6956         .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6957         .test_ring = gfx_v8_0_ring_test_ring,
6958         .test_ib = gfx_v8_0_ring_test_ib,
6959         .insert_nop = amdgpu_ring_insert_nop,
6960         .pad_ib = amdgpu_ring_generic_pad_ib,
6961         .emit_switch_buffer = gfx_v8_ring_emit_sb,
6962         .emit_cntxcntl = gfx_v8_ring_emit_cntxcntl,
6963         .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec,
6964         .patch_cond_exec = gfx_v8_0_ring_emit_patch_cond_exec,
6965         .emit_wreg = gfx_v8_0_ring_emit_wreg,
6966         .soft_recovery = gfx_v8_0_ring_soft_recovery,
6967         .emit_mem_sync = gfx_v8_0_emit_mem_sync,
6968 };
6969
6970 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
6971         .type = AMDGPU_RING_TYPE_COMPUTE,
6972         .align_mask = 0xff,
6973         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6974         .support_64bit_ptrs = false,
6975         .get_rptr = gfx_v8_0_ring_get_rptr,
6976         .get_wptr = gfx_v8_0_ring_get_wptr_compute,
6977         .set_wptr = gfx_v8_0_ring_set_wptr_compute,
6978         .emit_frame_size =
6979                 20 + /* gfx_v8_0_ring_emit_gds_switch */
6980                 7 + /* gfx_v8_0_ring_emit_hdp_flush */
6981                 5 + /* hdp_invalidate */
6982                 7 + /* gfx_v8_0_ring_emit_pipeline_sync */
6983                 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */
6984                 7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */
6985                 7 + /* gfx_v8_0_emit_mem_sync_compute */
6986                 5 + /* gfx_v8_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6987                 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6988         .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */
6989         .emit_ib = gfx_v8_0_ring_emit_ib_compute,
6990         .emit_fence = gfx_v8_0_ring_emit_fence_compute,
6991         .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
6992         .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
6993         .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
6994         .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6995         .test_ring = gfx_v8_0_ring_test_ring,
6996         .test_ib = gfx_v8_0_ring_test_ib,
6997         .insert_nop = amdgpu_ring_insert_nop,
6998         .pad_ib = amdgpu_ring_generic_pad_ib,
6999         .emit_wreg = gfx_v8_0_ring_emit_wreg,
7000         .emit_mem_sync = gfx_v8_0_emit_mem_sync_compute,
7001         .emit_wave_limit = gfx_v8_0_emit_wave_limit,
7002 };
7003
7004 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
7005         .type = AMDGPU_RING_TYPE_KIQ,
7006         .align_mask = 0xff,
7007         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7008         .support_64bit_ptrs = false,
7009         .get_rptr = gfx_v8_0_ring_get_rptr,
7010         .get_wptr = gfx_v8_0_ring_get_wptr_compute,
7011         .set_wptr = gfx_v8_0_ring_set_wptr_compute,
7012         .emit_frame_size =
7013                 20 + /* gfx_v8_0_ring_emit_gds_switch */
7014                 7 + /* gfx_v8_0_ring_emit_hdp_flush */
7015                 5 + /* hdp_invalidate */
7016                 7 + /* gfx_v8_0_ring_emit_pipeline_sync */
7017                 17 + /* gfx_v8_0_ring_emit_vm_flush */
7018                 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7019         .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */
7020         .emit_fence = gfx_v8_0_ring_emit_fence_kiq,
7021         .test_ring = gfx_v8_0_ring_test_ring,
7022         .insert_nop = amdgpu_ring_insert_nop,
7023         .pad_ib = amdgpu_ring_generic_pad_ib,
7024         .emit_rreg = gfx_v8_0_ring_emit_rreg,
7025         .emit_wreg = gfx_v8_0_ring_emit_wreg,
7026 };
7027
7028 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
7029 {
7030         int i;
7031
7032         adev->gfx.kiq.ring.funcs = &gfx_v8_0_ring_funcs_kiq;
7033
7034         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7035                 adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx;
7036
7037         for (i = 0; i < adev->gfx.num_compute_rings; i++)
7038                 adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute;
7039 }
7040
7041 static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = {
7042         .set = gfx_v8_0_set_eop_interrupt_state,
7043         .process = gfx_v8_0_eop_irq,
7044 };
7045
7046 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = {
7047         .set = gfx_v8_0_set_priv_reg_fault_state,
7048         .process = gfx_v8_0_priv_reg_irq,
7049 };
7050
7051 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = {
7052         .set = gfx_v8_0_set_priv_inst_fault_state,
7053         .process = gfx_v8_0_priv_inst_irq,
7054 };
7055
7056 static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = {
7057         .set = gfx_v8_0_set_cp_ecc_int_state,
7058         .process = gfx_v8_0_cp_ecc_error_irq,
7059 };
7060
7061 static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = {
7062         .set = gfx_v8_0_set_sq_int_state,
7063         .process = gfx_v8_0_sq_irq,
7064 };
7065
7066 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev)
7067 {
7068         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7069         adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs;
7070
7071         adev->gfx.priv_reg_irq.num_types = 1;
7072         adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs;
7073
7074         adev->gfx.priv_inst_irq.num_types = 1;
7075         adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs;
7076
7077         adev->gfx.cp_ecc_error_irq.num_types = 1;
7078         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs;
7079
7080         adev->gfx.sq_irq.num_types = 1;
7081         adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs;
7082 }
7083
7084 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)
7085 {
7086         adev->gfx.rlc.funcs = &iceland_rlc_funcs;
7087 }
7088
7089 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)
7090 {
7091         /* init asci gds info */
7092         adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE);
7093         adev->gds.gws_size = 64;
7094         adev->gds.oa_size = 16;
7095         adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID);
7096 }
7097
7098 static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7099                                                  u32 bitmap)
7100 {
7101         u32 data;
7102
7103         if (!bitmap)
7104                 return;
7105
7106         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7107         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7108
7109         WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
7110 }
7111
7112 static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7113 {
7114         u32 data, mask;
7115
7116         data =  RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) |
7117                 RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
7118
7119         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7120
7121         return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask;
7122 }
7123
7124 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
7125 {
7126         int i, j, k, counter, active_cu_number = 0;
7127         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7128         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
7129         unsigned disable_masks[4 * 2];
7130         u32 ao_cu_num;
7131
7132         memset(cu_info, 0, sizeof(*cu_info));
7133
7134         if (adev->flags & AMD_IS_APU)
7135                 ao_cu_num = 2;
7136         else
7137                 ao_cu_num = adev->gfx.config.max_cu_per_sh;
7138
7139         amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
7140
7141         mutex_lock(&adev->grbm_idx_mutex);
7142         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7143                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7144                         mask = 1;
7145                         ao_bitmap = 0;
7146                         counter = 0;
7147                         gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
7148                         if (i < 4 && j < 2)
7149                                 gfx_v8_0_set_user_cu_inactive_bitmap(
7150                                         adev, disable_masks[i * 2 + j]);
7151                         bitmap = gfx_v8_0_get_cu_active_bitmap(adev);
7152                         cu_info->bitmap[i][j] = bitmap;
7153
7154                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7155                                 if (bitmap & mask) {
7156                                         if (counter < ao_cu_num)
7157                                                 ao_bitmap |= mask;
7158                                         counter ++;
7159                                 }
7160                                 mask <<= 1;
7161                         }
7162                         active_cu_number += counter;
7163                         if (i < 2 && j < 2)
7164                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7165                         cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
7166                 }
7167         }
7168         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7169         mutex_unlock(&adev->grbm_idx_mutex);
7170
7171         cu_info->number = active_cu_number;
7172         cu_info->ao_cu_mask = ao_cu_mask;
7173         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7174         cu_info->max_waves_per_simd = 10;
7175         cu_info->max_scratch_slots_per_cu = 32;
7176         cu_info->wave_front_size = 64;
7177         cu_info->lds_size = 64;
7178 }
7179
7180 const struct amdgpu_ip_block_version gfx_v8_0_ip_block =
7181 {
7182         .type = AMD_IP_BLOCK_TYPE_GFX,
7183         .major = 8,
7184         .minor = 0,
7185         .rev = 0,
7186         .funcs = &gfx_v8_0_ip_funcs,
7187 };
7188
7189 const struct amdgpu_ip_block_version gfx_v8_1_ip_block =
7190 {
7191         .type = AMD_IP_BLOCK_TYPE_GFX,
7192         .major = 8,
7193         .minor = 1,
7194         .rev = 0,
7195         .funcs = &gfx_v8_0_ip_funcs,
7196 };
7197
7198 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
7199 {
7200         uint64_t ce_payload_addr;
7201         int cnt_ce;
7202         union {
7203                 struct vi_ce_ib_state regular;
7204                 struct vi_ce_ib_state_chained_ib chained;
7205         } ce_payload = {};
7206
7207         if (ring->adev->virt.chained_ib_support) {
7208                 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
7209                         offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
7210                 cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
7211         } else {
7212                 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
7213                         offsetof(struct vi_gfx_meta_data, ce_payload);
7214                 cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
7215         }
7216
7217         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce));
7218         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
7219                                 WRITE_DATA_DST_SEL(8) |
7220                                 WR_CONFIRM) |
7221                                 WRITE_DATA_CACHE_POLICY(0));
7222         amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr));
7223         amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr));
7224         amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2);
7225 }
7226
7227 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
7228 {
7229         uint64_t de_payload_addr, gds_addr, csa_addr;
7230         int cnt_de;
7231         union {
7232                 struct vi_de_ib_state regular;
7233                 struct vi_de_ib_state_chained_ib chained;
7234         } de_payload = {};
7235
7236         csa_addr = amdgpu_csa_vaddr(ring->adev);
7237         gds_addr = csa_addr + 4096;
7238         if (ring->adev->virt.chained_ib_support) {
7239                 de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
7240                 de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr);
7241                 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data_chained_ib, de_payload);
7242                 cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2;
7243         } else {
7244                 de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr);
7245                 de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr);
7246                 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data, de_payload);
7247                 cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2;
7248         }
7249
7250         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de));
7251         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
7252                                 WRITE_DATA_DST_SEL(8) |
7253                                 WR_CONFIRM) |
7254                                 WRITE_DATA_CACHE_POLICY(0));
7255         amdgpu_ring_write(ring, lower_32_bits(de_payload_addr));
7256         amdgpu_ring_write(ring, upper_32_bits(de_payload_addr));
7257         amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2);
7258 }