drivers/gpu/drm/amd/amdgpu/si.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "amdgpu.h"
  29 #include "amdgpu_atombios.h"
  30 #include "amdgpu_ih.h"
  31 #include "amdgpu_uvd.h"
  32 #include "amdgpu_vce.h"
  33 #include "atom.h"
  34 #include "amdgpu_powerplay.h"
  35 #include "sid.h"
  36 #include "si_ih.h"
  37 #include "gfx_v6_0.h"
  38 #include "gmc_v6_0.h"
  39 #include "si_dma.h"
  40 #include "dce_v6_0.h"
  41 #include "si.h"
  42 #include "dce_virtual.h"
  43 #include "gca/gfx_6_0_d.h"
  44 #include "oss/oss_1_0_d.h"
  45 #include "gmc/gmc_6_0_d.h"
  46 #include "dce/dce_6_0_d.h"
  47 #include "uvd/uvd_4_0_d.h"
  48
  49 static const u32 tahiti_golden_registers[] =
  50 {
  51         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
  52         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
  53         mmDB_DEBUG, 0xffffffff, 0x00000000,
  54         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
  55         mmDB_DEBUG3, 0x0002021c, 0x00020200,
  56         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
  57         0x340c, 0x000000c0, 0x00800040,
  58         0x360c, 0x000000c0, 0x00800040,
  59         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
  60         mmFBC_MISC, 0x00200000, 0x50100000,
  61         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
  62         mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
  63         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
  64         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
  65         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
  66         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
  67         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
  68         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
  69         0x000c, 0xffffffff, 0x0040,
  70         0x000d, 0x00000040, 0x00004040,
  71         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
  72         mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
  73         mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
  74         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
  75         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
  76         mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
  77         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
  78         mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
  79         mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
  80         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
  81         mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
  82         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
  83         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
  84         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  85         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  86         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  87 };
  88
  89 static const u32 tahiti_golden_registers2[] =
  90 {
  91         mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
  92 };
  93
  94 static const u32 tahiti_golden_rlc_registers[] =
  95 {
  96         mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
  97         mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
  98         0x311f, 0xffffffff, 0x10104040,
  99         0x3122, 0xffffffff, 0x0100000a,
 100         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 101         mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 102         mmUVD_CGC_GATE, 0x00000008, 0x00000000,
 103 };
 104
 105 static const u32 pitcairn_golden_registers[] =
 106 {
 107         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 108         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 109         mmDB_DEBUG, 0xffffffff, 0x00000000,
 110         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 111         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 112         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 113         0x340c, 0x000300c0, 0x00800040,
 114         0x360c, 0x000300c0, 0x00800040,
 115         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 116         mmFBC_MISC, 0x00200000, 0x50100000,
 117         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 118         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 119         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 120         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 121         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 122         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 123         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 124         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
 125         0x000c, 0xffffffff, 0x0040,
 126         0x000d, 0x00000040, 0x00004040,
 127         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 128         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 129         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 130         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 131         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 132         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
 133         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 134         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 135         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 136         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 137         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 138         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 139 };
 140
 141 static const u32 pitcairn_golden_rlc_registers[] =
 142 {
 143         mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 144         mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
 145         0x311f, 0xffffffff, 0x10102020,
 146         0x3122, 0xffffffff, 0x01000020,
 147         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 148         mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
 149 };
 150
 151 static const u32 verde_pg_init[] =
 152 {
 153         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
 154         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
 155         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 156         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 157         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 158         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 159         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 160         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
 161         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
 162         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 163         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 164         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 165         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 166         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 167         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
 168         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
 169         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 170         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 171         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 172         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 173         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 174         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
 175         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
 176         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 177         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 178         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 179         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 180         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 181         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
 182         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
 183         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 184         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 185         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 186         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 187         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 188         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
 189         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
 190         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 191         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 192         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 193         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 194         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 195         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 196         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
 197         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
 198         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
 199         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 200         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 201         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
 202         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
 203         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 204         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 205         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
 206         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
 207         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
 208         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
 209         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
 210         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
 211         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
 212         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
 213         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
 214         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
 215         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
 216         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
 217         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
 218         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
 219         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
 220         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
 221         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
 222         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
 223         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
 224         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
 225         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
 226         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
 227         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
 228         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
 229         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
 230         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
 231         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
 232         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
 233         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
 234         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
 235         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
 236         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
 237         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
 238         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
 239         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
 240         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
 241         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
 242         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
 243         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
 244         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
 245         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
 246         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
 247         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
 248         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
 249         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
 250         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
 251         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
 252         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
 253         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
 254         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
 255         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
 256         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
 257         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
 258         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
 259         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
 260         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
 261         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
 262         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
 263         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
 264         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
 265         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
 266         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
 267         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
 268         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
 269         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
 270         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
 271         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
 272         mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
 273         mmGMCON_MISC2, 0xfc00, 0x2000,
 274         mmGMCON_MISC3, 0xffffffff, 0xfc0,
 275         mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
 276 };
 277
 278 static const u32 verde_golden_rlc_registers[] =
 279 {
 280         mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 281         mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
 282         0x311f, 0xffffffff, 0x10808020,
 283         0x3122, 0xffffffff, 0x00800008,
 284         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
 285         mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
 286 };
 287
 288 static const u32 verde_golden_registers[] =
 289 {
 290         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 291         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 292         mmDB_DEBUG, 0xffffffff, 0x00000000,
 293         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 294         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 295         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 296         0x340c, 0x000300c0, 0x00800040,
 297         0x360c, 0x000300c0, 0x00800040,
 298         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 299         mmFBC_MISC, 0x00200000, 0x50100000,
 300         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 301         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 302         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 303         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 304         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 305         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 306         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 307         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
 308         0x000c, 0xffffffff, 0x0040,
 309         0x000d, 0x00000040, 0x00004040,
 310         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 311         mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
 312         mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
 313         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 314         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 315         mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
 316         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 317         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
 318         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 319         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 320         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 321         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 322         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 323         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 324 };
 325
 326 static const u32 oland_golden_registers[] =
 327 {
 328         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 329         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 330         mmDB_DEBUG, 0xffffffff, 0x00000000,
 331         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 332         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 333         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 334         0x340c, 0x000300c0, 0x00800040,
 335         0x360c, 0x000300c0, 0x00800040,
 336         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 337         mmFBC_MISC, 0x00200000, 0x50100000,
 338         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 339         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 340         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 341         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 342         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 343         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 344         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 345         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
 346         0x000c, 0xffffffff, 0x0040,
 347         0x000d, 0x00000040, 0x00004040,
 348         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 349         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 350         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 351         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
 352         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 353         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 354         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 355         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 356         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 357         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 358         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 359         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 360
 361 };
 362
 363 static const u32 oland_golden_rlc_registers[] =
 364 {
 365         mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 366         mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 367         0x311f, 0xffffffff, 0x10104040,
 368         0x3122, 0xffffffff, 0x0100000a,
 369         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 370         mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 371 };
 372
 373 static const u32 hainan_golden_registers[] =
 374 {
 375         0x17bc, 0x00000030, 0x00000011,
 376         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 377         mmDB_DEBUG, 0xffffffff, 0x00000000,
 378         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 379         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 380         0x031e, 0x00000080, 0x00000000,
 381         0x3430, 0xff000fff, 0x00000100,
 382         0x340c, 0x000300c0, 0x00800040,
 383         0x3630, 0xff000fff, 0x00000100,
 384         0x360c, 0x000300c0, 0x00800040,
 385         0x16ec, 0x000000f0, 0x00000070,
 386         0x16f0, 0x00200000, 0x50100000,
 387         0x1c0c, 0x31000311, 0x00000011,
 388         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 389         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 390         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 391         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 392         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 393         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 394         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
 395         0x000c, 0xffffffff, 0x0040,
 396         0x000d, 0x00000040, 0x00004040,
 397         mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
 398         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 399         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 400         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
 401         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 402         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 403         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 404         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 405         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 406         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 407         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 408         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 409 };
 410
 411 static const u32 hainan_golden_registers2[] =
 412 {
 413         mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
 414 };
 415
 416 static const u32 tahiti_mgcg_cgcg_init[] =
 417 {
 418         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 419         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 420         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 421         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 422         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 423         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 424         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 425         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 426         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 427         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 428         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 429         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 430         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 431         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 432         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 433         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 434         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 435         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 436         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 437         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 438         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 439         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 440         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 441         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 442         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 443         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 444         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 445         0x2458, 0xffffffff, 0x00010000,
 446         0x2459, 0xffffffff, 0x00030002,
 447         0x245a, 0xffffffff, 0x00040007,
 448         0x245b, 0xffffffff, 0x00060005,
 449         0x245c, 0xffffffff, 0x00090008,
 450         0x245d, 0xffffffff, 0x00020001,
 451         0x245e, 0xffffffff, 0x00040003,
 452         0x245f, 0xffffffff, 0x00000007,
 453         0x2460, 0xffffffff, 0x00060005,
 454         0x2461, 0xffffffff, 0x00090008,
 455         0x2462, 0xffffffff, 0x00030002,
 456         0x2463, 0xffffffff, 0x00050004,
 457         0x2464, 0xffffffff, 0x00000008,
 458         0x2465, 0xffffffff, 0x00070006,
 459         0x2466, 0xffffffff, 0x000a0009,
 460         0x2467, 0xffffffff, 0x00040003,
 461         0x2468, 0xffffffff, 0x00060005,
 462         0x2469, 0xffffffff, 0x00000009,
 463         0x246a, 0xffffffff, 0x00080007,
 464         0x246b, 0xffffffff, 0x000b000a,
 465         0x246c, 0xffffffff, 0x00050004,
 466         0x246d, 0xffffffff, 0x00070006,
 467         0x246e, 0xffffffff, 0x0008000b,
 468         0x246f, 0xffffffff, 0x000a0009,
 469         0x2470, 0xffffffff, 0x000d000c,
 470         0x2471, 0xffffffff, 0x00060005,
 471         0x2472, 0xffffffff, 0x00080007,
 472         0x2473, 0xffffffff, 0x0000000b,
 473         0x2474, 0xffffffff, 0x000a0009,
 474         0x2475, 0xffffffff, 0x000d000c,
 475         0x2476, 0xffffffff, 0x00070006,
 476         0x2477, 0xffffffff, 0x00090008,
 477         0x2478, 0xffffffff, 0x0000000c,
 478         0x2479, 0xffffffff, 0x000b000a,
 479         0x247a, 0xffffffff, 0x000e000d,
 480         0x247b, 0xffffffff, 0x00080007,
 481         0x247c, 0xffffffff, 0x000a0009,
 482         0x247d, 0xffffffff, 0x0000000d,
 483         0x247e, 0xffffffff, 0x000c000b,
 484         0x247f, 0xffffffff, 0x000f000e,
 485         0x2480, 0xffffffff, 0x00090008,
 486         0x2481, 0xffffffff, 0x000b000a,
 487         0x2482, 0xffffffff, 0x000c000f,
 488         0x2483, 0xffffffff, 0x000e000d,
 489         0x2484, 0xffffffff, 0x00110010,
 490         0x2485, 0xffffffff, 0x000a0009,
 491         0x2486, 0xffffffff, 0x000c000b,
 492         0x2487, 0xffffffff, 0x0000000f,
 493         0x2488, 0xffffffff, 0x000e000d,
 494         0x2489, 0xffffffff, 0x00110010,
 495         0x248a, 0xffffffff, 0x000b000a,
 496         0x248b, 0xffffffff, 0x000d000c,
 497         0x248c, 0xffffffff, 0x00000010,
 498         0x248d, 0xffffffff, 0x000f000e,
 499         0x248e, 0xffffffff, 0x00120011,
 500         0x248f, 0xffffffff, 0x000c000b,
 501         0x2490, 0xffffffff, 0x000e000d,
 502         0x2491, 0xffffffff, 0x00000011,
 503         0x2492, 0xffffffff, 0x0010000f,
 504         0x2493, 0xffffffff, 0x00130012,
 505         0x2494, 0xffffffff, 0x000d000c,
 506         0x2495, 0xffffffff, 0x000f000e,
 507         0x2496, 0xffffffff, 0x00100013,
 508         0x2497, 0xffffffff, 0x00120011,
 509         0x2498, 0xffffffff, 0x00150014,
 510         0x2499, 0xffffffff, 0x000e000d,
 511         0x249a, 0xffffffff, 0x0010000f,
 512         0x249b, 0xffffffff, 0x00000013,
 513         0x249c, 0xffffffff, 0x00120011,
 514         0x249d, 0xffffffff, 0x00150014,
 515         0x249e, 0xffffffff, 0x000f000e,
 516         0x249f, 0xffffffff, 0x00110010,
 517         0x24a0, 0xffffffff, 0x00000014,
 518         0x24a1, 0xffffffff, 0x00130012,
 519         0x24a2, 0xffffffff, 0x00160015,
 520         0x24a3, 0xffffffff, 0x0010000f,
 521         0x24a4, 0xffffffff, 0x00120011,
 522         0x24a5, 0xffffffff, 0x00000015,
 523         0x24a6, 0xffffffff, 0x00140013,
 524         0x24a7, 0xffffffff, 0x00170016,
 525         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 526         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 527         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 528         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 529         0x000c, 0xffffffff, 0x0000001c,
 530         0x000d, 0x000f0000, 0x000f0000,
 531         0x0583, 0xffffffff, 0x00000100,
 532         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 533         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 534         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 535         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 536         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 537         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 538         0x157a, 0x00000001, 0x00000001,
 539         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 540         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 541         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 542         0x3430, 0xfffffff0, 0x00000100,
 543         0x3630, 0xfffffff0, 0x00000100,
 544 };
 545 static const u32 pitcairn_mgcg_cgcg_init[] =
 546 {
 547         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 548         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 549         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 550         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 551         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 552         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 553         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 554         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 555         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 556         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 557         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 558         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 559         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 560         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 561         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 562         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 563         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 564         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 565         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 566         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 567         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 568         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 569         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 570         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 571         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 572         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 573         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 574         0x2458, 0xffffffff, 0x00010000,
 575         0x2459, 0xffffffff, 0x00030002,
 576         0x245a, 0xffffffff, 0x00040007,
 577         0x245b, 0xffffffff, 0x00060005,
 578         0x245c, 0xffffffff, 0x00090008,
 579         0x245d, 0xffffffff, 0x00020001,
 580         0x245e, 0xffffffff, 0x00040003,
 581         0x245f, 0xffffffff, 0x00000007,
 582         0x2460, 0xffffffff, 0x00060005,
 583         0x2461, 0xffffffff, 0x00090008,
 584         0x2462, 0xffffffff, 0x00030002,
 585         0x2463, 0xffffffff, 0x00050004,
 586         0x2464, 0xffffffff, 0x00000008,
 587         0x2465, 0xffffffff, 0x00070006,
 588         0x2466, 0xffffffff, 0x000a0009,
 589         0x2467, 0xffffffff, 0x00040003,
 590         0x2468, 0xffffffff, 0x00060005,
 591         0x2469, 0xffffffff, 0x00000009,
 592         0x246a, 0xffffffff, 0x00080007,
 593         0x246b, 0xffffffff, 0x000b000a,
 594         0x246c, 0xffffffff, 0x00050004,
 595         0x246d, 0xffffffff, 0x00070006,
 596         0x246e, 0xffffffff, 0x0008000b,
 597         0x246f, 0xffffffff, 0x000a0009,
 598         0x2470, 0xffffffff, 0x000d000c,
 599         0x2480, 0xffffffff, 0x00090008,
 600         0x2481, 0xffffffff, 0x000b000a,
 601         0x2482, 0xffffffff, 0x000c000f,
 602         0x2483, 0xffffffff, 0x000e000d,
 603         0x2484, 0xffffffff, 0x00110010,
 604         0x2485, 0xffffffff, 0x000a0009,
 605         0x2486, 0xffffffff, 0x000c000b,
 606         0x2487, 0xffffffff, 0x0000000f,
 607         0x2488, 0xffffffff, 0x000e000d,
 608         0x2489, 0xffffffff, 0x00110010,
 609         0x248a, 0xffffffff, 0x000b000a,
 610         0x248b, 0xffffffff, 0x000d000c,
 611         0x248c, 0xffffffff, 0x00000010,
 612         0x248d, 0xffffffff, 0x000f000e,
 613         0x248e, 0xffffffff, 0x00120011,
 614         0x248f, 0xffffffff, 0x000c000b,
 615         0x2490, 0xffffffff, 0x000e000d,
 616         0x2491, 0xffffffff, 0x00000011,
 617         0x2492, 0xffffffff, 0x0010000f,
 618         0x2493, 0xffffffff, 0x00130012,
 619         0x2494, 0xffffffff, 0x000d000c,
 620         0x2495, 0xffffffff, 0x000f000e,
 621         0x2496, 0xffffffff, 0x00100013,
 622         0x2497, 0xffffffff, 0x00120011,
 623         0x2498, 0xffffffff, 0x00150014,
 624         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 625         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 626         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 627         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 628         0x000c, 0xffffffff, 0x0000001c,
 629         0x000d, 0x000f0000, 0x000f0000,
 630         0x0583, 0xffffffff, 0x00000100,
 631         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 632         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 633         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 634         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 635         0x157a, 0x00000001, 0x00000001,
 636         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 637         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 638         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 639         0x3430, 0xfffffff0, 0x00000100,
 640         0x3630, 0xfffffff0, 0x00000100,
 641 };
 642
 643 static const u32 verde_mgcg_cgcg_init[] =
 644 {
 645         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 646         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 647         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 648         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 649         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 650         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 651         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 652         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 653         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 654         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 655         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 656         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 657         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 658         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 659         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 660         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 661         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 662         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 663         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 664         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 665         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 666         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 667         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 668         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 669         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 670         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 671         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 672         0x2458, 0xffffffff, 0x00010000,
 673         0x2459, 0xffffffff, 0x00030002,
 674         0x245a, 0xffffffff, 0x00040007,
 675         0x245b, 0xffffffff, 0x00060005,
 676         0x245c, 0xffffffff, 0x00090008,
 677         0x245d, 0xffffffff, 0x00020001,
 678         0x245e, 0xffffffff, 0x00040003,
 679         0x245f, 0xffffffff, 0x00000007,
 680         0x2460, 0xffffffff, 0x00060005,
 681         0x2461, 0xffffffff, 0x00090008,
 682         0x2462, 0xffffffff, 0x00030002,
 683         0x2463, 0xffffffff, 0x00050004,
 684         0x2464, 0xffffffff, 0x00000008,
 685         0x2465, 0xffffffff, 0x00070006,
 686         0x2466, 0xffffffff, 0x000a0009,
 687         0x2467, 0xffffffff, 0x00040003,
 688         0x2468, 0xffffffff, 0x00060005,
 689         0x2469, 0xffffffff, 0x00000009,
 690         0x246a, 0xffffffff, 0x00080007,
 691         0x246b, 0xffffffff, 0x000b000a,
 692         0x246c, 0xffffffff, 0x00050004,
 693         0x246d, 0xffffffff, 0x00070006,
 694         0x246e, 0xffffffff, 0x0008000b,
 695         0x246f, 0xffffffff, 0x000a0009,
 696         0x2470, 0xffffffff, 0x000d000c,
 697         0x2480, 0xffffffff, 0x00090008,
 698         0x2481, 0xffffffff, 0x000b000a,
 699         0x2482, 0xffffffff, 0x000c000f,
 700         0x2483, 0xffffffff, 0x000e000d,
 701         0x2484, 0xffffffff, 0x00110010,
 702         0x2485, 0xffffffff, 0x000a0009,
 703         0x2486, 0xffffffff, 0x000c000b,
 704         0x2487, 0xffffffff, 0x0000000f,
 705         0x2488, 0xffffffff, 0x000e000d,
 706         0x2489, 0xffffffff, 0x00110010,
 707         0x248a, 0xffffffff, 0x000b000a,
 708         0x248b, 0xffffffff, 0x000d000c,
 709         0x248c, 0xffffffff, 0x00000010,
 710         0x248d, 0xffffffff, 0x000f000e,
 711         0x248e, 0xffffffff, 0x00120011,
 712         0x248f, 0xffffffff, 0x000c000b,
 713         0x2490, 0xffffffff, 0x000e000d,
 714         0x2491, 0xffffffff, 0x00000011,
 715         0x2492, 0xffffffff, 0x0010000f,
 716         0x2493, 0xffffffff, 0x00130012,
 717         0x2494, 0xffffffff, 0x000d000c,
 718         0x2495, 0xffffffff, 0x000f000e,
 719         0x2496, 0xffffffff, 0x00100013,
 720         0x2497, 0xffffffff, 0x00120011,
 721         0x2498, 0xffffffff, 0x00150014,
 722         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 723         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 724         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 725         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 726         0x000c, 0xffffffff, 0x0000001c,
 727         0x000d, 0x000f0000, 0x000f0000,
 728         0x0583, 0xffffffff, 0x00000100,
 729         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 730         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 731         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 732         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 733         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 734         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 735         0x157a, 0x00000001, 0x00000001,
 736         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 737         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 738         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 739         0x3430, 0xfffffff0, 0x00000100,
 740         0x3630, 0xfffffff0, 0x00000100,
 741 };
 742
 743 static const u32 oland_mgcg_cgcg_init[] =
 744 {
 745         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 746         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 747         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 748         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 749         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 750         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 751         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 752         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 753         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 754         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 755         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 756         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 757         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 758         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 759         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 760         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 761         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 762         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 763         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 764         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 765         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 766         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 767         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 768         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 769         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 770         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 771         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 772         0x2458, 0xffffffff, 0x00010000,
 773         0x2459, 0xffffffff, 0x00030002,
 774         0x245a, 0xffffffff, 0x00040007,
 775         0x245b, 0xffffffff, 0x00060005,
 776         0x245c, 0xffffffff, 0x00090008,
 777         0x245d, 0xffffffff, 0x00020001,
 778         0x245e, 0xffffffff, 0x00040003,
 779         0x245f, 0xffffffff, 0x00000007,
 780         0x2460, 0xffffffff, 0x00060005,
 781         0x2461, 0xffffffff, 0x00090008,
 782         0x2462, 0xffffffff, 0x00030002,
 783         0x2463, 0xffffffff, 0x00050004,
 784         0x2464, 0xffffffff, 0x00000008,
 785         0x2465, 0xffffffff, 0x00070006,
 786         0x2466, 0xffffffff, 0x000a0009,
 787         0x2467, 0xffffffff, 0x00040003,
 788         0x2468, 0xffffffff, 0x00060005,
 789         0x2469, 0xffffffff, 0x00000009,
 790         0x246a, 0xffffffff, 0x00080007,
 791         0x246b, 0xffffffff, 0x000b000a,
 792         0x246c, 0xffffffff, 0x00050004,
 793         0x246d, 0xffffffff, 0x00070006,
 794         0x246e, 0xffffffff, 0x0008000b,
 795         0x246f, 0xffffffff, 0x000a0009,
 796         0x2470, 0xffffffff, 0x000d000c,
 797         0x2471, 0xffffffff, 0x00060005,
 798         0x2472, 0xffffffff, 0x00080007,
 799         0x2473, 0xffffffff, 0x0000000b,
 800         0x2474, 0xffffffff, 0x000a0009,
 801         0x2475, 0xffffffff, 0x000d000c,
 802         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 803         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 804         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 805         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 806         0x000c, 0xffffffff, 0x0000001c,
 807         0x000d, 0x000f0000, 0x000f0000,
 808         0x0583, 0xffffffff, 0x00000100,
 809         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 810         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 811         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 812         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 813         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 814         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 815         0x157a, 0x00000001, 0x00000001,
 816         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 817         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 818         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 819         0x3430, 0xfffffff0, 0x00000100,
 820         0x3630, 0xfffffff0, 0x00000100,
 821 };
 822
 823 static const u32 hainan_mgcg_cgcg_init[] =
 824 {
 825         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 826         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 827         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 828         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 829         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 830         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 831         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 832         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 833         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 834         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 835         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 836         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 837         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 838         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 839         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 840         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 841         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 842         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 843         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 844         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 845         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 846         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 847         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 848         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 849         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 850         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 851         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 852         0x2458, 0xffffffff, 0x00010000,
 853         0x2459, 0xffffffff, 0x00030002,
 854         0x245a, 0xffffffff, 0x00040007,
 855         0x245b, 0xffffffff, 0x00060005,
 856         0x245c, 0xffffffff, 0x00090008,
 857         0x245d, 0xffffffff, 0x00020001,
 858         0x245e, 0xffffffff, 0x00040003,
 859         0x245f, 0xffffffff, 0x00000007,
 860         0x2460, 0xffffffff, 0x00060005,
 861         0x2461, 0xffffffff, 0x00090008,
 862         0x2462, 0xffffffff, 0x00030002,
 863         0x2463, 0xffffffff, 0x00050004,
 864         0x2464, 0xffffffff, 0x00000008,
 865         0x2465, 0xffffffff, 0x00070006,
 866         0x2466, 0xffffffff, 0x000a0009,
 867         0x2467, 0xffffffff, 0x00040003,
 868         0x2468, 0xffffffff, 0x00060005,
 869         0x2469, 0xffffffff, 0x00000009,
 870         0x246a, 0xffffffff, 0x00080007,
 871         0x246b, 0xffffffff, 0x000b000a,
 872         0x246c, 0xffffffff, 0x00050004,
 873         0x246d, 0xffffffff, 0x00070006,
 874         0x246e, 0xffffffff, 0x0008000b,
 875         0x246f, 0xffffffff, 0x000a0009,
 876         0x2470, 0xffffffff, 0x000d000c,
 877         0x2471, 0xffffffff, 0x00060005,
 878         0x2472, 0xffffffff, 0x00080007,
 879         0x2473, 0xffffffff, 0x0000000b,
 880         0x2474, 0xffffffff, 0x000a0009,
 881         0x2475, 0xffffffff, 0x000d000c,
 882         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 883         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 884         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 885         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 886         0x000c, 0xffffffff, 0x0000001c,
 887         0x000d, 0x000f0000, 0x000f0000,
 888         0x0583, 0xffffffff, 0x00000100,
 889         0x0409, 0xffffffff, 0x00000100,
 890         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 891         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 892         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 893         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 894         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 895         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 896         0x3430, 0xfffffff0, 0x00000100,
 897         0x3630, 0xfffffff0, 0x00000100,
 898 };
 899
 900 static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 901 {
 902         unsigned long flags;
 903         u32 r;
 904
 905         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 906         WREG32(AMDGPU_PCIE_INDEX, reg);
 907         (void)RREG32(AMDGPU_PCIE_INDEX);
 908         r = RREG32(AMDGPU_PCIE_DATA);
 909         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 910         return r;
 911 }
 912
 913 static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 914 {
 915         unsigned long flags;
 916
 917         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 918         WREG32(AMDGPU_PCIE_INDEX, reg);
 919         (void)RREG32(AMDGPU_PCIE_INDEX);
 920         WREG32(AMDGPU_PCIE_DATA, v);
 921         (void)RREG32(AMDGPU_PCIE_DATA);
 922         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 923 }
 924
 925 static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
 926 {
 927         unsigned long flags;
 928         u32 r;
 929
 930         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 931         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 932         (void)RREG32(PCIE_PORT_INDEX);
 933         r = RREG32(PCIE_PORT_DATA);
 934         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 935         return r;
 936 }
 937
 938 static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 939 {
 940         unsigned long flags;
 941
 942         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 943         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 944         (void)RREG32(PCIE_PORT_INDEX);
 945         WREG32(PCIE_PORT_DATA, (v));
 946         (void)RREG32(PCIE_PORT_DATA);
 947         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 948 }
 949
 950 static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
 951 {
 952         unsigned long flags;
 953         u32 r;
 954
 955         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 956         WREG32(SMC_IND_INDEX_0, (reg));
 957         r = RREG32(SMC_IND_DATA_0);
 958         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 959         return r;
 960 }
 961
 962 static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 963 {
 964         unsigned long flags;
 965
 966         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 967         WREG32(SMC_IND_INDEX_0, (reg));
 968         WREG32(SMC_IND_DATA_0, (v));
 969         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 970 }
 971
 972 static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
 973         {GRBM_STATUS, false},
 974         {GB_ADDR_CONFIG, false},
 975         {MC_ARB_RAMCFG, false},
 976         {GB_TILE_MODE0, false},
 977         {GB_TILE_MODE1, false},
 978         {GB_TILE_MODE2, false},
 979         {GB_TILE_MODE3, false},
 980         {GB_TILE_MODE4, false},
 981         {GB_TILE_MODE5, false},
 982         {GB_TILE_MODE6, false},
 983         {GB_TILE_MODE7, false},
 984         {GB_TILE_MODE8, false},
 985         {GB_TILE_MODE9, false},
 986         {GB_TILE_MODE10, false},
 987         {GB_TILE_MODE11, false},
 988         {GB_TILE_MODE12, false},
 989         {GB_TILE_MODE13, false},
 990         {GB_TILE_MODE14, false},
 991         {GB_TILE_MODE15, false},
 992         {GB_TILE_MODE16, false},
 993         {GB_TILE_MODE17, false},
 994         {GB_TILE_MODE18, false},
 995         {GB_TILE_MODE19, false},
 996         {GB_TILE_MODE20, false},
 997         {GB_TILE_MODE21, false},
 998         {GB_TILE_MODE22, false},
 999         {GB_TILE_MODE23, false},
1000         {GB_TILE_MODE24, false},
1001         {GB_TILE_MODE25, false},
1002         {GB_TILE_MODE26, false},
1003         {GB_TILE_MODE27, false},
1004         {GB_TILE_MODE28, false},
1005         {GB_TILE_MODE29, false},
1006         {GB_TILE_MODE30, false},
1007         {GB_TILE_MODE31, false},
1008         {CC_RB_BACKEND_DISABLE, false, true},
1009         {GC_USER_RB_BACKEND_DISABLE, false, true},
1010         {PA_SC_RASTER_CONFIG, false, true},
1011 };
1012
1013 static uint32_t si_get_register_value(struct amdgpu_device *adev,
1014                                       bool indexed, u32 se_num,
1015                                       u32 sh_num, u32 reg_offset)
1016 {
1017         if (indexed) {
1018                 uint32_t val;
1019                 unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1020                 unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1021
1022                 switch (reg_offset) {
1023                 case mmCC_RB_BACKEND_DISABLE:
1024                         return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1025                 case mmGC_USER_RB_BACKEND_DISABLE:
1026                         return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1027                 case mmPA_SC_RASTER_CONFIG:
1028                         return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1029                 }
1030
1031                 mutex_lock(&adev->grbm_idx_mutex);
1032                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1033                         amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1034
1035                 val = RREG32(reg_offset);
1036
1037                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1038                         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1039                 mutex_unlock(&adev->grbm_idx_mutex);
1040                 return val;
1041         } else {
1042                 unsigned idx;
1043
1044                 switch (reg_offset) {
1045                 case mmGB_ADDR_CONFIG:
1046                         return adev->gfx.config.gb_addr_config;
1047                 case mmMC_ARB_RAMCFG:
1048                         return adev->gfx.config.mc_arb_ramcfg;
1049                 case mmGB_TILE_MODE0:
1050                 case mmGB_TILE_MODE1:
1051                 case mmGB_TILE_MODE2:
1052                 case mmGB_TILE_MODE3:
1053                 case mmGB_TILE_MODE4:
1054                 case mmGB_TILE_MODE5:
1055                 case mmGB_TILE_MODE6:
1056                 case mmGB_TILE_MODE7:
1057                 case mmGB_TILE_MODE8:
1058                 case mmGB_TILE_MODE9:
1059                 case mmGB_TILE_MODE10:
1060                 case mmGB_TILE_MODE11:
1061                 case mmGB_TILE_MODE12:
1062                 case mmGB_TILE_MODE13:
1063                 case mmGB_TILE_MODE14:
1064                 case mmGB_TILE_MODE15:
1065                 case mmGB_TILE_MODE16:
1066                 case mmGB_TILE_MODE17:
1067                 case mmGB_TILE_MODE18:
1068                 case mmGB_TILE_MODE19:
1069                 case mmGB_TILE_MODE20:
1070                 case mmGB_TILE_MODE21:
1071                 case mmGB_TILE_MODE22:
1072                 case mmGB_TILE_MODE23:
1073                 case mmGB_TILE_MODE24:
1074                 case mmGB_TILE_MODE25:
1075                 case mmGB_TILE_MODE26:
1076                 case mmGB_TILE_MODE27:
1077                 case mmGB_TILE_MODE28:
1078                 case mmGB_TILE_MODE29:
1079                 case mmGB_TILE_MODE30:
1080                 case mmGB_TILE_MODE31:
1081                         idx = (reg_offset - mmGB_TILE_MODE0);
1082                         return adev->gfx.config.tile_mode_array[idx];
1083                 default:
1084                         return RREG32(reg_offset);
1085                 }
1086         }
1087 }
1088 static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1089                              u32 sh_num, u32 reg_offset, u32 *value)
1090 {
1091         uint32_t i;
1092
1093         *value = 0;
1094         for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1095                 if (reg_offset != si_allowed_read_registers[i].reg_offset)
1096                         continue;
1097
1098                 if (!si_allowed_read_registers[i].untouched)
1099                         *value = si_get_register_value(adev,
1100                                                 si_allowed_read_registers[i].grbm_indexed,
1101                                                 se_num, sh_num, reg_offset);
1102                 return 0;
1103         }
1104         return -EINVAL;
1105 }
1106
1107 static bool si_read_disabled_bios(struct amdgpu_device *adev)
1108 {
1109         u32 bus_cntl;
1110         u32 d1vga_control = 0;
1111         u32 d2vga_control = 0;
1112         u32 vga_render_control = 0;
1113         u32 rom_cntl;
1114         bool r;
1115
1116         bus_cntl = RREG32(R600_BUS_CNTL);
1117         if (adev->mode_info.num_crtc) {
1118                 d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
1119                 d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
1120                 vga_render_control = RREG32(VGA_RENDER_CONTROL);
1121         }
1122         rom_cntl = RREG32(R600_ROM_CNTL);
1123
1124         /* enable the rom */
1125         WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
1126         if (adev->mode_info.num_crtc) {
1127                 /* Disable VGA mode */
1128                 WREG32(AVIVO_D1VGA_CONTROL,
1129                        (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1130                                           AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1131                 WREG32(AVIVO_D2VGA_CONTROL,
1132                        (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1133                                           AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1134                 WREG32(VGA_RENDER_CONTROL,
1135                        (vga_render_control & C_000300_VGA_VSTATUS_CNTL));
1136         }
1137         WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1138
1139         r = amdgpu_read_bios(adev);
1140
1141         /* restore regs */
1142         WREG32(R600_BUS_CNTL, bus_cntl);
1143         if (adev->mode_info.num_crtc) {
1144                 WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
1145                 WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
1146                 WREG32(VGA_RENDER_CONTROL, vga_render_control);
1147         }
1148         WREG32(R600_ROM_CNTL, rom_cntl);
1149         return r;
1150 }
1151
1152 //xxx: not implemented
1153 static int si_asic_reset(struct amdgpu_device *adev)
1154 {
1155         return 0;
1156 }
1157
1158 static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1159 {
1160         uint32_t temp;
1161
1162         temp = RREG32(CONFIG_CNTL);
1163         if (state == false) {
1164                 temp &= ~(1<<0);
1165                 temp |= (1<<1);
1166         } else {
1167                 temp &= ~(1<<1);
1168         }
1169         WREG32(CONFIG_CNTL, temp);
1170 }
1171
1172 static u32 si_get_xclk(struct amdgpu_device *adev)
1173 {
1174         u32 reference_clock = adev->clock.spll.reference_freq;
1175         u32 tmp;
1176
1177         tmp = RREG32(CG_CLKPIN_CNTL_2);
1178         if (tmp & MUX_TCLK_TO_XCLK)
1179                 return TCLK;
1180
1181         tmp = RREG32(CG_CLKPIN_CNTL);
1182         if (tmp & XTALIN_DIVIDE)
1183                 return reference_clock / 4;
1184
1185         return reference_clock;
1186 }
1187
1188 //xxx:not implemented
1189 static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1190 {
1191         return 0;
1192 }
1193
1194 static void si_detect_hw_virtualization(struct amdgpu_device *adev)
1195 {
1196         if (is_virtual_machine()) /* passthrough mode */
1197                 adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
1198 }
1199
1200 static const struct amdgpu_asic_funcs si_asic_funcs =
1201 {
1202         .read_disabled_bios = &si_read_disabled_bios,
1203         .read_register = &si_read_register,
1204         .reset = &si_asic_reset,
1205         .set_vga_state = &si_vga_set_state,
1206         .get_xclk = &si_get_xclk,
1207         .set_uvd_clocks = &si_set_uvd_clocks,
1208         .set_vce_clocks = NULL,
1209 };
1210
1211 static uint32_t si_get_rev_id(struct amdgpu_device *adev)
1212 {
1213         return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1214                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1215 }
1216
1217 static int si_common_early_init(void *handle)
1218 {
1219         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1220
1221         adev->smc_rreg = &si_smc_rreg;
1222         adev->smc_wreg = &si_smc_wreg;
1223         adev->pcie_rreg = &si_pcie_rreg;
1224         adev->pcie_wreg = &si_pcie_wreg;
1225         adev->pciep_rreg = &si_pciep_rreg;
1226         adev->pciep_wreg = &si_pciep_wreg;
1227         adev->uvd_ctx_rreg = NULL;
1228         adev->uvd_ctx_wreg = NULL;
1229         adev->didt_rreg = NULL;
1230         adev->didt_wreg = NULL;
1231
1232         adev->asic_funcs = &si_asic_funcs;
1233
1234         adev->rev_id = si_get_rev_id(adev);
1235         adev->external_rev_id = 0xFF;
1236         switch (adev->asic_type) {
1237         case CHIP_TAHITI:
1238                 adev->cg_flags =
1239                         AMD_CG_SUPPORT_GFX_MGCG |
1240                         AMD_CG_SUPPORT_GFX_MGLS |
1241                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1242                         AMD_CG_SUPPORT_GFX_CGLS |
1243                         AMD_CG_SUPPORT_GFX_CGTS |
1244                         AMD_CG_SUPPORT_GFX_CP_LS |
1245                         AMD_CG_SUPPORT_MC_MGCG |
1246                         AMD_CG_SUPPORT_SDMA_MGCG |
1247                         AMD_CG_SUPPORT_BIF_LS |
1248                         AMD_CG_SUPPORT_VCE_MGCG |
1249                         AMD_CG_SUPPORT_UVD_MGCG |
1250                         AMD_CG_SUPPORT_HDP_LS |
1251                         AMD_CG_SUPPORT_HDP_MGCG;
1252                         adev->pg_flags = 0;
1253                 adev->external_rev_id = (adev->rev_id == 0) ? 1 :
1254                                         (adev->rev_id == 1) ? 5 : 6;
1255                 break;
1256         case CHIP_PITCAIRN:
1257                 adev->cg_flags =
1258                         AMD_CG_SUPPORT_GFX_MGCG |
1259                         AMD_CG_SUPPORT_GFX_MGLS |
1260                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1261                         AMD_CG_SUPPORT_GFX_CGLS |
1262                         AMD_CG_SUPPORT_GFX_CGTS |
1263                         AMD_CG_SUPPORT_GFX_CP_LS |
1264                         AMD_CG_SUPPORT_GFX_RLC_LS |
1265                         AMD_CG_SUPPORT_MC_LS |
1266                         AMD_CG_SUPPORT_MC_MGCG |
1267                         AMD_CG_SUPPORT_SDMA_MGCG |
1268                         AMD_CG_SUPPORT_BIF_LS |
1269                         AMD_CG_SUPPORT_VCE_MGCG |
1270                         AMD_CG_SUPPORT_UVD_MGCG |
1271                         AMD_CG_SUPPORT_HDP_LS |
1272                         AMD_CG_SUPPORT_HDP_MGCG;
1273                 adev->pg_flags = 0;
1274                 adev->external_rev_id = adev->rev_id + 20;
1275                 break;
1276
1277         case CHIP_VERDE:
1278                 adev->cg_flags =
1279                         AMD_CG_SUPPORT_GFX_MGCG |
1280                         AMD_CG_SUPPORT_GFX_MGLS |
1281                         AMD_CG_SUPPORT_GFX_CGLS |
1282                         AMD_CG_SUPPORT_GFX_CGTS |
1283                         AMD_CG_SUPPORT_GFX_CGTS_LS |
1284                         AMD_CG_SUPPORT_GFX_CP_LS |
1285                         AMD_CG_SUPPORT_MC_LS |
1286                         AMD_CG_SUPPORT_MC_MGCG |
1287                         AMD_CG_SUPPORT_SDMA_MGCG |
1288                         AMD_CG_SUPPORT_SDMA_LS |
1289                         AMD_CG_SUPPORT_BIF_LS |
1290                         AMD_CG_SUPPORT_VCE_MGCG |
1291                         AMD_CG_SUPPORT_UVD_MGCG |
1292                         AMD_CG_SUPPORT_HDP_LS |
1293                         AMD_CG_SUPPORT_HDP_MGCG;
1294                 adev->pg_flags = 0;
1295                 //???
1296                 adev->external_rev_id = adev->rev_id + 40;
1297                 break;
1298         case CHIP_OLAND:
1299                 adev->cg_flags =
1300                         AMD_CG_SUPPORT_GFX_MGCG |
1301                         AMD_CG_SUPPORT_GFX_MGLS |
1302                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1303                         AMD_CG_SUPPORT_GFX_CGLS |
1304                         AMD_CG_SUPPORT_GFX_CGTS |
1305                         AMD_CG_SUPPORT_GFX_CP_LS |
1306                         AMD_CG_SUPPORT_GFX_RLC_LS |
1307                         AMD_CG_SUPPORT_MC_LS |
1308                         AMD_CG_SUPPORT_MC_MGCG |
1309                         AMD_CG_SUPPORT_SDMA_MGCG |
1310                         AMD_CG_SUPPORT_BIF_LS |
1311                         AMD_CG_SUPPORT_UVD_MGCG |
1312                         AMD_CG_SUPPORT_HDP_LS |
1313                         AMD_CG_SUPPORT_HDP_MGCG;
1314                 adev->pg_flags = 0;
1315                 adev->external_rev_id = 60;
1316                 break;
1317         case CHIP_HAINAN:
1318                 adev->cg_flags =
1319                         AMD_CG_SUPPORT_GFX_MGCG |
1320                         AMD_CG_SUPPORT_GFX_MGLS |
1321                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1322                         AMD_CG_SUPPORT_GFX_CGLS |
1323                         AMD_CG_SUPPORT_GFX_CGTS |
1324                         AMD_CG_SUPPORT_GFX_CP_LS |
1325                         AMD_CG_SUPPORT_GFX_RLC_LS |
1326                         AMD_CG_SUPPORT_MC_LS |
1327                         AMD_CG_SUPPORT_MC_MGCG |
1328                         AMD_CG_SUPPORT_SDMA_MGCG |
1329                         AMD_CG_SUPPORT_BIF_LS |
1330                         AMD_CG_SUPPORT_HDP_LS |
1331                         AMD_CG_SUPPORT_HDP_MGCG;
1332                 adev->pg_flags = 0;
1333                 adev->external_rev_id = 70;
1334                 break;
1335
1336         default:
1337                 return -EINVAL;
1338         }
1339
1340         return 0;
1341 }
1342
1343 static int si_common_sw_init(void *handle)
1344 {
1345         return 0;
1346 }
1347
1348 static int si_common_sw_fini(void *handle)
1349 {
1350         return 0;
1351 }
1352
1353
1354 static void si_init_golden_registers(struct amdgpu_device *adev)
1355 {
1356         switch (adev->asic_type) {
1357         case CHIP_TAHITI:
1358                 amdgpu_program_register_sequence(adev,
1359                                                  tahiti_golden_registers,
1360                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers));
1361                 amdgpu_program_register_sequence(adev,
1362                                                  tahiti_golden_rlc_registers,
1363                                                  (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1364                 amdgpu_program_register_sequence(adev,
1365                                                  tahiti_mgcg_cgcg_init,
1366                                                  (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1367                 amdgpu_program_register_sequence(adev,
1368                                                  tahiti_golden_registers2,
1369                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1370                 break;
1371         case CHIP_PITCAIRN:
1372                 amdgpu_program_register_sequence(adev,
1373                                                  pitcairn_golden_registers,
1374                                                  (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1375                 amdgpu_program_register_sequence(adev,
1376                                                  pitcairn_golden_rlc_registers,
1377                                                  (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1378                 amdgpu_program_register_sequence(adev,
1379                                                  pitcairn_mgcg_cgcg_init,
1380                                                  (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1381         case CHIP_VERDE:
1382                 amdgpu_program_register_sequence(adev,
1383                                                  verde_golden_registers,
1384                                                  (const u32)ARRAY_SIZE(verde_golden_registers));
1385                 amdgpu_program_register_sequence(adev,
1386                                                  verde_golden_rlc_registers,
1387                                                  (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1388                 amdgpu_program_register_sequence(adev,
1389                                                  verde_mgcg_cgcg_init,
1390                                                  (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1391                 amdgpu_program_register_sequence(adev,
1392                                                  verde_pg_init,
1393                                                  (const u32)ARRAY_SIZE(verde_pg_init));
1394                 break;
1395         case CHIP_OLAND:
1396                 amdgpu_program_register_sequence(adev,
1397                                                  oland_golden_registers,
1398                                                  (const u32)ARRAY_SIZE(oland_golden_registers));
1399                 amdgpu_program_register_sequence(adev,
1400                                                  oland_golden_rlc_registers,
1401                                                  (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1402                 amdgpu_program_register_sequence(adev,
1403                                                  oland_mgcg_cgcg_init,
1404                                                  (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1405         case CHIP_HAINAN:
1406                 amdgpu_program_register_sequence(adev,
1407                                                  hainan_golden_registers,
1408                                                  (const u32)ARRAY_SIZE(hainan_golden_registers));
1409                 amdgpu_program_register_sequence(adev,
1410                                                  hainan_golden_registers2,
1411                                                  (const u32)ARRAY_SIZE(hainan_golden_registers2));
1412                 amdgpu_program_register_sequence(adev,
1413                                                  hainan_mgcg_cgcg_init,
1414                                                  (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1415                 break;
1416
1417
1418         default:
1419                 BUG();
1420         }
1421 }
1422
1423 static void si_pcie_gen3_enable(struct amdgpu_device *adev)
1424 {
1425         struct pci_dev *root = adev->pdev->bus->self;
1426         int bridge_pos, gpu_pos;
1427         u32 speed_cntl, mask, current_data_rate;
1428         int ret, i;
1429         u16 tmp16;
1430
1431         if (pci_is_root_bus(adev->pdev->bus))
1432                 return;
1433
1434         if (amdgpu_pcie_gen2 == 0)
1435                 return;
1436
1437         if (adev->flags & AMD_IS_APU)
1438                 return;
1439
1440         ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
1441         if (ret != 0)
1442                 return;
1443
1444         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
1445                 return;
1446
1447         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1448         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
1449                 LC_CURRENT_DATA_RATE_SHIFT;
1450         if (mask & DRM_PCIE_SPEED_80) {
1451                 if (current_data_rate == 2) {
1452                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1453                         return;
1454                 }
1455                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1456         } else if (mask & DRM_PCIE_SPEED_50) {
1457                 if (current_data_rate == 1) {
1458                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1459                         return;
1460                 }
1461                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1462         }
1463
1464         bridge_pos = pci_pcie_cap(root);
1465         if (!bridge_pos)
1466                 return;
1467
1468         gpu_pos = pci_pcie_cap(adev->pdev);
1469         if (!gpu_pos)
1470                 return;
1471
1472         if (mask & DRM_PCIE_SPEED_80) {
1473                 if (current_data_rate != 2) {
1474                         u16 bridge_cfg, gpu_cfg;
1475                         u16 bridge_cfg2, gpu_cfg2;
1476                         u32 max_lw, current_lw, tmp;
1477
1478                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1479                         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1480
1481                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1482                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1483
1484                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1485                         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1486
1487                         tmp = RREG32_PCIE(PCIE_LC_STATUS1);
1488                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
1489                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
1490
1491                         if (current_lw < max_lw) {
1492                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1493                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
1494                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
1495                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
1496                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
1497                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
1498                                 }
1499                         }
1500
1501                         for (i = 0; i < 10; i++) {
1502                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1503                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1504                                         break;
1505
1506                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1507                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1508
1509                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1510                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1511
1512                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1513                                 tmp |= LC_SET_QUIESCE;
1514                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1515
1516                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1517                                 tmp |= LC_REDO_EQ;
1518                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1519
1520                                 mdelay(100);
1521
1522                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1523                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1524                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1525                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1526
1527                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1528                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1529                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1530                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1531
1532                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1533                                 tmp16 &= ~((1 << 4) | (7 << 9));
1534                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1535                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1536
1537                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1538                                 tmp16 &= ~((1 << 4) | (7 << 9));
1539                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1540                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1541
1542                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1543                                 tmp &= ~LC_SET_QUIESCE;
1544                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1545                         }
1546                 }
1547         }
1548
1549         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
1550         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
1551         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1552
1553         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1554         tmp16 &= ~0xf;
1555         if (mask & DRM_PCIE_SPEED_80)
1556                 tmp16 |= 3;
1557         else if (mask & DRM_PCIE_SPEED_50)
1558                 tmp16 |= 2;
1559         else
1560                 tmp16 |= 1;
1561         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1562
1563         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1564         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
1565         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1566
1567         for (i = 0; i < adev->usec_timeout; i++) {
1568                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1569                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
1570                         break;
1571                 udelay(1);
1572         }
1573 }
1574
1575 static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
1576 {
1577         unsigned long flags;
1578         u32 r;
1579
1580         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1581         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1582         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
1583         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1584         return r;
1585 }
1586
1587 static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1588 {
1589         unsigned long flags;
1590
1591         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1592         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1593         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
1594         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1595 }
1596
1597 static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
1598 {
1599         unsigned long flags;
1600         u32 r;
1601
1602         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1603         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1604         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
1605         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1606         return r;
1607 }
1608
1609 static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1610 {
1611         unsigned long flags;
1612
1613         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1614         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1615         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
1616         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1617 }
1618 static void si_program_aspm(struct amdgpu_device *adev)
1619 {
1620         u32 data, orig;
1621         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1622         bool disable_clkreq = false;
1623
1624         if (amdgpu_aspm == 0)
1625                 return;
1626
1627         if (adev->flags & AMD_IS_APU)
1628                 return;
1629         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
1630         data &= ~LC_XMIT_N_FTS_MASK;
1631         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
1632         if (orig != data)
1633                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
1634
1635         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
1636         data |= LC_GO_TO_RECOVERY;
1637         if (orig != data)
1638                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
1639
1640         orig = data = RREG32_PCIE(PCIE_P_CNTL);
1641         data |= P_IGNORE_EDB_ERR;
1642         if (orig != data)
1643                 WREG32_PCIE(PCIE_P_CNTL, data);
1644
1645         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
1646         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
1647         data |= LC_PMI_TO_L1_DIS;
1648         if (!disable_l0s)
1649                 data |= LC_L0S_INACTIVITY(7);
1650
1651         if (!disable_l1) {
1652                 data |= LC_L1_INACTIVITY(7);
1653                 data &= ~LC_PMI_TO_L1_DIS;
1654                 if (orig != data)
1655                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1656
1657                 if (!disable_plloff_in_l1) {
1658                         bool clk_req_support;
1659
1660                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1661                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1662                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1663                         if (orig != data)
1664                                 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1665
1666                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1667                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1668                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1669                         if (orig != data)
1670                                 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1671
1672                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1673                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1674                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1675                         if (orig != data)
1676                                 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1677
1678                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1679                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1680                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1681                         if (orig != data)
1682                                 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1683
1684                         if ((adev->family != CHIP_OLAND) && (adev->family != CHIP_HAINAN)) {
1685                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1686                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
1687                                 if (orig != data)
1688                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1689
1690                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1691                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
1692                                 if (orig != data)
1693                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1694
1695                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);
1696                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
1697                                 if (orig != data)
1698                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);
1699
1700                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);
1701                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
1702                                 if (orig != data)
1703                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);
1704
1705                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1706                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
1707                                 if (orig != data)
1708                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1709
1710                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1711                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
1712                                 if (orig != data)
1713                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1714
1715                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);
1716                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
1717                                 if (orig != data)
1718                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);
1719
1720                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);
1721                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
1722                                 if (orig != data)
1723                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);
1724                         }
1725                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1726                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
1727                         data |= LC_DYN_LANES_PWR_STATE(3);
1728                         if (orig != data)
1729                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
1730
1731                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);
1732                         data &= ~LS2_EXIT_TIME_MASK;
1733                         if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
1734                                 data |= LS2_EXIT_TIME(5);
1735                         if (orig != data)
1736                                 si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);
1737
1738                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);
1739                         data &= ~LS2_EXIT_TIME_MASK;
1740                         if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
1741                                 data |= LS2_EXIT_TIME(5);
1742                         if (orig != data)
1743                                 si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);
1744
1745                         if (!disable_clkreq &&
1746                             !pci_is_root_bus(adev->pdev->bus)) {
1747                                 struct pci_dev *root = adev->pdev->bus->self;
1748                                 u32 lnkcap;
1749
1750                                 clk_req_support = false;
1751                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1752                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1753                                         clk_req_support = true;
1754                         } else {
1755                                 clk_req_support = false;
1756                         }
1757
1758                         if (clk_req_support) {
1759                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
1760                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
1761                                 if (orig != data)
1762                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
1763
1764                                 orig = data = RREG32(THM_CLK_CNTL);
1765                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
1766                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
1767                                 if (orig != data)
1768                                         WREG32(THM_CLK_CNTL, data);
1769
1770                                 orig = data = RREG32(MISC_CLK_CNTL);
1771                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
1772                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
1773                                 if (orig != data)
1774                                         WREG32(MISC_CLK_CNTL, data);
1775
1776                                 orig = data = RREG32(CG_CLKPIN_CNTL);
1777                                 data &= ~BCLK_AS_XCLK;
1778                                 if (orig != data)
1779                                         WREG32(CG_CLKPIN_CNTL, data);
1780
1781                                 orig = data = RREG32(CG_CLKPIN_CNTL_2);
1782                                 data &= ~FORCE_BIF_REFCLK_EN;
1783                                 if (orig != data)
1784                                         WREG32(CG_CLKPIN_CNTL_2, data);
1785
1786                                 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
1787                                 data &= ~MPLL_CLKOUT_SEL_MASK;
1788                                 data |= MPLL_CLKOUT_SEL(4);
1789                                 if (orig != data)
1790                                         WREG32(MPLL_BYPASSCLK_SEL, data);
1791
1792                                 orig = data = RREG32(SPLL_CNTL_MODE);
1793                                 data &= ~SPLL_REFCLK_SEL_MASK;
1794                                 if (orig != data)
1795                                         WREG32(SPLL_CNTL_MODE, data);
1796                         }
1797                 }
1798         } else {
1799                 if (orig != data)
1800                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1801         }
1802
1803         orig = data = RREG32_PCIE(PCIE_CNTL2);
1804         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
1805         if (orig != data)
1806                 WREG32_PCIE(PCIE_CNTL2, data);
1807
1808         if (!disable_l0s) {
1809                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
1810                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
1811                         data = RREG32_PCIE(PCIE_LC_STATUS1);
1812                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
1813                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
1814                                 data &= ~LC_L0S_INACTIVITY_MASK;
1815                                 if (orig != data)
1816                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1817                         }
1818                 }
1819         }
1820 }
1821
1822 static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
1823 {
1824         int readrq;
1825         u16 v;
1826
1827         readrq = pcie_get_readrq(adev->pdev);
1828         v = ffs(readrq) - 8;
1829         if ((v == 0) || (v == 6) || (v == 7))
1830                 pcie_set_readrq(adev->pdev, 512);
1831 }
1832
1833 static int si_common_hw_init(void *handle)
1834 {
1835         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1836
1837         si_fix_pci_max_read_req_size(adev);
1838         si_init_golden_registers(adev);
1839         si_pcie_gen3_enable(adev);
1840         si_program_aspm(adev);
1841
1842         return 0;
1843 }
1844
1845 static int si_common_hw_fini(void *handle)
1846 {
1847         return 0;
1848 }
1849
1850 static int si_common_suspend(void *handle)
1851 {
1852         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1853
1854         return si_common_hw_fini(adev);
1855 }
1856
1857 static int si_common_resume(void *handle)
1858 {
1859         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1860
1861         return si_common_hw_init(adev);
1862 }
1863
1864 static bool si_common_is_idle(void *handle)
1865 {
1866         return true;
1867 }
1868
1869 static int si_common_wait_for_idle(void *handle)
1870 {
1871         return 0;
1872 }
1873
1874 static int si_common_soft_reset(void *handle)
1875 {
1876         return 0;
1877 }
1878
1879 static int si_common_set_clockgating_state(void *handle,
1880                                             enum amd_clockgating_state state)
1881 {
1882         return 0;
1883 }
1884
1885 static int si_common_set_powergating_state(void *handle,
1886                                             enum amd_powergating_state state)
1887 {
1888         return 0;
1889 }
1890
1891 static const struct amd_ip_funcs si_common_ip_funcs = {
1892         .name = "si_common",
1893         .early_init = si_common_early_init,
1894         .late_init = NULL,
1895         .sw_init = si_common_sw_init,
1896         .sw_fini = si_common_sw_fini,
1897         .hw_init = si_common_hw_init,
1898         .hw_fini = si_common_hw_fini,
1899         .suspend = si_common_suspend,
1900         .resume = si_common_resume,
1901         .is_idle = si_common_is_idle,
1902         .wait_for_idle = si_common_wait_for_idle,
1903         .soft_reset = si_common_soft_reset,
1904         .set_clockgating_state = si_common_set_clockgating_state,
1905         .set_powergating_state = si_common_set_powergating_state,
1906 };
1907
1908 static const struct amdgpu_ip_block_version si_common_ip_block =
1909 {
1910         .type = AMD_IP_BLOCK_TYPE_COMMON,
1911         .major = 1,
1912         .minor = 0,
1913         .rev = 0,
1914         .funcs = &si_common_ip_funcs,
1915 };
1916
1917 int si_set_ip_blocks(struct amdgpu_device *adev)
1918 {
1919         si_detect_hw_virtualization(adev);
1920
1921         switch (adev->asic_type) {
1922         case CHIP_VERDE:
1923         case CHIP_TAHITI:
1924         case CHIP_PITCAIRN:
1925                 amdgpu_ip_block_add(adev, &si_common_ip_block);
1926                 amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
1927                 amdgpu_ip_block_add(adev, &si_ih_ip_block);
1928                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
1929                 if (adev->enable_virtual_display)
1930                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
1931                 else
1932                         amdgpu_ip_block_add(adev, &dce_v6_0_ip_block);
1933                 amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
1934                 amdgpu_ip_block_add(adev, &si_dma_ip_block);
1935                 /* amdgpu_ip_block_add(adev, &uvd_v3_1_ip_block); */
1936                 /* amdgpu_ip_block_add(adev, &vce_v1_0_ip_block); */
1937                 break;
1938         case CHIP_OLAND:
1939                 amdgpu_ip_block_add(adev, &si_common_ip_block);
1940                 amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
1941                 amdgpu_ip_block_add(adev, &si_ih_ip_block);
1942                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
1943                 if (adev->enable_virtual_display)
1944                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
1945                 else
1946                         amdgpu_ip_block_add(adev, &dce_v6_4_ip_block);
1947                 amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
1948                 amdgpu_ip_block_add(adev, &si_dma_ip_block);
1949                 /* amdgpu_ip_block_add(adev, &uvd_v3_1_ip_block); */
1950                 /* amdgpu_ip_block_add(adev, &vce_v1_0_ip_block); */
1951                 break;
1952         case CHIP_HAINAN:
1953                 amdgpu_ip_block_add(adev, &si_common_ip_block);
1954                 amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
1955                 amdgpu_ip_block_add(adev, &si_ih_ip_block);
1956                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
1957                 if (adev->enable_virtual_display)
1958                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
1959                 amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
1960                 amdgpu_ip_block_add(adev, &si_dma_ip_block);
1961                 break;
1962         default:
1963                 BUG();
1964         }
1965         return 0;
1966 }
1967