drivers/gpu/drm/amd/amdgpu/si.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_atombios.h"
  30 #include "amdgpu_ih.h"
  31 #include "amdgpu_uvd.h"
  32 #include "amdgpu_vce.h"
  33 #include "atom.h"
  34 #include "amdgpu_powerplay.h"
  35 #include "sid.h"
  36 #include "si_ih.h"
  37 #include "gfx_v6_0.h"
  38 #include "gmc_v6_0.h"
  39 #include "si_dma.h"
  40 #include "dce_v6_0.h"
  41 #include "si.h"
  42 #include "dce_virtual.h"
  43 #include "gca/gfx_6_0_d.h"
  44 #include "oss/oss_1_0_d.h"
  45 #include "gmc/gmc_6_0_d.h"
  46 #include "dce/dce_6_0_d.h"
  47 #include "uvd/uvd_4_0_d.h"
  48 #include "bif/bif_3_0_d.h"
  49
  50 static const u32 tahiti_golden_registers[] =
  51 {
  52         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
  53         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
  54         mmDB_DEBUG, 0xffffffff, 0x00000000,
  55         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
  56         mmDB_DEBUG3, 0x0002021c, 0x00020200,
  57         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
  58         0x340c, 0x000000c0, 0x00800040,
  59         0x360c, 0x000000c0, 0x00800040,
  60         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
  61         mmFBC_MISC, 0x00200000, 0x50100000,
  62         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
  63         mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
  64         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
  65         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
  66         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
  67         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
  68         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
  69         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
  70         0x000c, 0xffffffff, 0x0040,
  71         0x000d, 0x00000040, 0x00004040,
  72         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
  73         mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
  74         mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
  75         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
  76         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
  77         mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
  78         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
  79         mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
  80         mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
  81         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
  82         mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
  83         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
  84         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
  85         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  86         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  87         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  88 };
  89
  90 static const u32 tahiti_golden_registers2[] =
  91 {
  92         mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
  93 };
  94
  95 static const u32 tahiti_golden_rlc_registers[] =
  96 {
  97         mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
  98         mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
  99         0x311f, 0xffffffff, 0x10104040,
 100         0x3122, 0xffffffff, 0x0100000a,
 101         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 102         mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 103         mmUVD_CGC_GATE, 0x00000008, 0x00000000,
 104 };
 105
 106 static const u32 pitcairn_golden_registers[] =
 107 {
 108         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 109         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 110         mmDB_DEBUG, 0xffffffff, 0x00000000,
 111         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 112         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 113         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 114         0x340c, 0x000300c0, 0x00800040,
 115         0x360c, 0x000300c0, 0x00800040,
 116         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 117         mmFBC_MISC, 0x00200000, 0x50100000,
 118         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 119         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 120         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 121         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 122         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 123         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 124         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 125         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
 126         0x000c, 0xffffffff, 0x0040,
 127         0x000d, 0x00000040, 0x00004040,
 128         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 129         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 130         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 131         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 132         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 133         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
 134         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 135         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 136         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 137         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 138         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 139         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 140 };
 141
 142 static const u32 pitcairn_golden_rlc_registers[] =
 143 {
 144         mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 145         mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
 146         0x311f, 0xffffffff, 0x10102020,
 147         0x3122, 0xffffffff, 0x01000020,
 148         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 149         mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
 150 };
 151
 152 static const u32 verde_pg_init[] =
 153 {
 154         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
 155         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
 156         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 157         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 158         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 159         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 160         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 161         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
 162         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
 163         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 164         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 165         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 166         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 167         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 168         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
 169         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
 170         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 171         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 172         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 173         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 174         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 175         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
 176         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
 177         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 178         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 179         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 180         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 181         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 182         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
 183         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
 184         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 185         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 186         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 187         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 188         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 189         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
 190         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
 191         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 192         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 193         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 194         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 195         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 196         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 197         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
 198         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
 199         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
 200         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 201         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 202         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
 203         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
 204         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 205         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 206         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
 207         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
 208         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
 209         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
 210         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
 211         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
 212         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
 213         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
 214         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
 215         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
 216         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
 217         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
 218         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
 219         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
 220         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
 221         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
 222         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
 223         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
 224         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
 225         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
 226         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
 227         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
 228         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
 229         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
 230         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
 231         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
 232         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
 233         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
 234         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
 235         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
 236         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
 237         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
 238         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
 239         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
 240         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
 241         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
 242         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
 243         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
 244         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
 245         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
 246         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
 247         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
 248         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
 249         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
 250         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
 251         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
 252         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
 253         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
 254         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
 255         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
 256         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
 257         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
 258         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
 259         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
 260         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
 261         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
 262         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
 263         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
 264         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
 265         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
 266         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
 267         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
 268         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
 269         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
 270         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
 271         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
 272         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
 273         mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
 274         mmGMCON_MISC2, 0xfc00, 0x2000,
 275         mmGMCON_MISC3, 0xffffffff, 0xfc0,
 276         mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
 277 };
 278
 279 static const u32 verde_golden_rlc_registers[] =
 280 {
 281         mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 282         mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
 283         0x311f, 0xffffffff, 0x10808020,
 284         0x3122, 0xffffffff, 0x00800008,
 285         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
 286         mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
 287 };
 288
 289 static const u32 verde_golden_registers[] =
 290 {
 291         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 292         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 293         mmDB_DEBUG, 0xffffffff, 0x00000000,
 294         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 295         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 296         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 297         0x340c, 0x000300c0, 0x00800040,
 298         0x360c, 0x000300c0, 0x00800040,
 299         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 300         mmFBC_MISC, 0x00200000, 0x50100000,
 301         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 302         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 303         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 304         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 305         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 306         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 307         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 308         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
 309         0x000c, 0xffffffff, 0x0040,
 310         0x000d, 0x00000040, 0x00004040,
 311         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 312         mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
 313         mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
 314         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 315         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 316         mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
 317         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 318         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
 319         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 320         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 321         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 322         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 323         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 324         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 325 };
 326
 327 static const u32 oland_golden_registers[] =
 328 {
 329         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 330         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 331         mmDB_DEBUG, 0xffffffff, 0x00000000,
 332         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 333         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 334         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 335         0x340c, 0x000300c0, 0x00800040,
 336         0x360c, 0x000300c0, 0x00800040,
 337         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 338         mmFBC_MISC, 0x00200000, 0x50100000,
 339         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 340         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 341         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 342         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 343         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 344         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 345         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 346         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
 347         0x000c, 0xffffffff, 0x0040,
 348         0x000d, 0x00000040, 0x00004040,
 349         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 350         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 351         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 352         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
 353         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 354         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 355         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 356         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 357         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 358         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 359         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 360         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 361
 362 };
 363
 364 static const u32 oland_golden_rlc_registers[] =
 365 {
 366         mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 367         mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 368         0x311f, 0xffffffff, 0x10104040,
 369         0x3122, 0xffffffff, 0x0100000a,
 370         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 371         mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 372 };
 373
 374 static const u32 hainan_golden_registers[] =
 375 {
 376         0x17bc, 0x00000030, 0x00000011,
 377         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 378         mmDB_DEBUG, 0xffffffff, 0x00000000,
 379         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 380         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 381         0x031e, 0x00000080, 0x00000000,
 382         0x3430, 0xff000fff, 0x00000100,
 383         0x340c, 0x000300c0, 0x00800040,
 384         0x3630, 0xff000fff, 0x00000100,
 385         0x360c, 0x000300c0, 0x00800040,
 386         0x16ec, 0x000000f0, 0x00000070,
 387         0x16f0, 0x00200000, 0x50100000,
 388         0x1c0c, 0x31000311, 0x00000011,
 389         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 390         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 391         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 392         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 393         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 394         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 395         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
 396         0x000c, 0xffffffff, 0x0040,
 397         0x000d, 0x00000040, 0x00004040,
 398         mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
 399         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 400         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 401         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
 402         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 403         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 404         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 405         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 406         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 407         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 408         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 409         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 410 };
 411
 412 static const u32 hainan_golden_registers2[] =
 413 {
 414         mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
 415 };
 416
 417 static const u32 tahiti_mgcg_cgcg_init[] =
 418 {
 419         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 420         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 421         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 422         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 423         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 424         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 425         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 426         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 427         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 428         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 429         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 430         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 431         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 432         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 433         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 434         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 435         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 436         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 437         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 438         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 439         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 440         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 441         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 442         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 443         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 444         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 445         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 446         0x2458, 0xffffffff, 0x00010000,
 447         0x2459, 0xffffffff, 0x00030002,
 448         0x245a, 0xffffffff, 0x00040007,
 449         0x245b, 0xffffffff, 0x00060005,
 450         0x245c, 0xffffffff, 0x00090008,
 451         0x245d, 0xffffffff, 0x00020001,
 452         0x245e, 0xffffffff, 0x00040003,
 453         0x245f, 0xffffffff, 0x00000007,
 454         0x2460, 0xffffffff, 0x00060005,
 455         0x2461, 0xffffffff, 0x00090008,
 456         0x2462, 0xffffffff, 0x00030002,
 457         0x2463, 0xffffffff, 0x00050004,
 458         0x2464, 0xffffffff, 0x00000008,
 459         0x2465, 0xffffffff, 0x00070006,
 460         0x2466, 0xffffffff, 0x000a0009,
 461         0x2467, 0xffffffff, 0x00040003,
 462         0x2468, 0xffffffff, 0x00060005,
 463         0x2469, 0xffffffff, 0x00000009,
 464         0x246a, 0xffffffff, 0x00080007,
 465         0x246b, 0xffffffff, 0x000b000a,
 466         0x246c, 0xffffffff, 0x00050004,
 467         0x246d, 0xffffffff, 0x00070006,
 468         0x246e, 0xffffffff, 0x0008000b,
 469         0x246f, 0xffffffff, 0x000a0009,
 470         0x2470, 0xffffffff, 0x000d000c,
 471         0x2471, 0xffffffff, 0x00060005,
 472         0x2472, 0xffffffff, 0x00080007,
 473         0x2473, 0xffffffff, 0x0000000b,
 474         0x2474, 0xffffffff, 0x000a0009,
 475         0x2475, 0xffffffff, 0x000d000c,
 476         0x2476, 0xffffffff, 0x00070006,
 477         0x2477, 0xffffffff, 0x00090008,
 478         0x2478, 0xffffffff, 0x0000000c,
 479         0x2479, 0xffffffff, 0x000b000a,
 480         0x247a, 0xffffffff, 0x000e000d,
 481         0x247b, 0xffffffff, 0x00080007,
 482         0x247c, 0xffffffff, 0x000a0009,
 483         0x247d, 0xffffffff, 0x0000000d,
 484         0x247e, 0xffffffff, 0x000c000b,
 485         0x247f, 0xffffffff, 0x000f000e,
 486         0x2480, 0xffffffff, 0x00090008,
 487         0x2481, 0xffffffff, 0x000b000a,
 488         0x2482, 0xffffffff, 0x000c000f,
 489         0x2483, 0xffffffff, 0x000e000d,
 490         0x2484, 0xffffffff, 0x00110010,
 491         0x2485, 0xffffffff, 0x000a0009,
 492         0x2486, 0xffffffff, 0x000c000b,
 493         0x2487, 0xffffffff, 0x0000000f,
 494         0x2488, 0xffffffff, 0x000e000d,
 495         0x2489, 0xffffffff, 0x00110010,
 496         0x248a, 0xffffffff, 0x000b000a,
 497         0x248b, 0xffffffff, 0x000d000c,
 498         0x248c, 0xffffffff, 0x00000010,
 499         0x248d, 0xffffffff, 0x000f000e,
 500         0x248e, 0xffffffff, 0x00120011,
 501         0x248f, 0xffffffff, 0x000c000b,
 502         0x2490, 0xffffffff, 0x000e000d,
 503         0x2491, 0xffffffff, 0x00000011,
 504         0x2492, 0xffffffff, 0x0010000f,
 505         0x2493, 0xffffffff, 0x00130012,
 506         0x2494, 0xffffffff, 0x000d000c,
 507         0x2495, 0xffffffff, 0x000f000e,
 508         0x2496, 0xffffffff, 0x00100013,
 509         0x2497, 0xffffffff, 0x00120011,
 510         0x2498, 0xffffffff, 0x00150014,
 511         0x2499, 0xffffffff, 0x000e000d,
 512         0x249a, 0xffffffff, 0x0010000f,
 513         0x249b, 0xffffffff, 0x00000013,
 514         0x249c, 0xffffffff, 0x00120011,
 515         0x249d, 0xffffffff, 0x00150014,
 516         0x249e, 0xffffffff, 0x000f000e,
 517         0x249f, 0xffffffff, 0x00110010,
 518         0x24a0, 0xffffffff, 0x00000014,
 519         0x24a1, 0xffffffff, 0x00130012,
 520         0x24a2, 0xffffffff, 0x00160015,
 521         0x24a3, 0xffffffff, 0x0010000f,
 522         0x24a4, 0xffffffff, 0x00120011,
 523         0x24a5, 0xffffffff, 0x00000015,
 524         0x24a6, 0xffffffff, 0x00140013,
 525         0x24a7, 0xffffffff, 0x00170016,
 526         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 527         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 528         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 529         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 530         0x000c, 0xffffffff, 0x0000001c,
 531         0x000d, 0x000f0000, 0x000f0000,
 532         0x0583, 0xffffffff, 0x00000100,
 533         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 534         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 535         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 536         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 537         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 538         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 539         0x157a, 0x00000001, 0x00000001,
 540         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 541         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 542         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 543         0x3430, 0xfffffff0, 0x00000100,
 544         0x3630, 0xfffffff0, 0x00000100,
 545 };
 546 static const u32 pitcairn_mgcg_cgcg_init[] =
 547 {
 548         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 549         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 550         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 551         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 552         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 553         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 554         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 555         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 556         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 557         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 558         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 559         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 560         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 561         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 562         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 563         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 564         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 565         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 566         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 567         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 568         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 569         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 570         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 571         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 572         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 573         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 574         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 575         0x2458, 0xffffffff, 0x00010000,
 576         0x2459, 0xffffffff, 0x00030002,
 577         0x245a, 0xffffffff, 0x00040007,
 578         0x245b, 0xffffffff, 0x00060005,
 579         0x245c, 0xffffffff, 0x00090008,
 580         0x245d, 0xffffffff, 0x00020001,
 581         0x245e, 0xffffffff, 0x00040003,
 582         0x245f, 0xffffffff, 0x00000007,
 583         0x2460, 0xffffffff, 0x00060005,
 584         0x2461, 0xffffffff, 0x00090008,
 585         0x2462, 0xffffffff, 0x00030002,
 586         0x2463, 0xffffffff, 0x00050004,
 587         0x2464, 0xffffffff, 0x00000008,
 588         0x2465, 0xffffffff, 0x00070006,
 589         0x2466, 0xffffffff, 0x000a0009,
 590         0x2467, 0xffffffff, 0x00040003,
 591         0x2468, 0xffffffff, 0x00060005,
 592         0x2469, 0xffffffff, 0x00000009,
 593         0x246a, 0xffffffff, 0x00080007,
 594         0x246b, 0xffffffff, 0x000b000a,
 595         0x246c, 0xffffffff, 0x00050004,
 596         0x246d, 0xffffffff, 0x00070006,
 597         0x246e, 0xffffffff, 0x0008000b,
 598         0x246f, 0xffffffff, 0x000a0009,
 599         0x2470, 0xffffffff, 0x000d000c,
 600         0x2480, 0xffffffff, 0x00090008,
 601         0x2481, 0xffffffff, 0x000b000a,
 602         0x2482, 0xffffffff, 0x000c000f,
 603         0x2483, 0xffffffff, 0x000e000d,
 604         0x2484, 0xffffffff, 0x00110010,
 605         0x2485, 0xffffffff, 0x000a0009,
 606         0x2486, 0xffffffff, 0x000c000b,
 607         0x2487, 0xffffffff, 0x0000000f,
 608         0x2488, 0xffffffff, 0x000e000d,
 609         0x2489, 0xffffffff, 0x00110010,
 610         0x248a, 0xffffffff, 0x000b000a,
 611         0x248b, 0xffffffff, 0x000d000c,
 612         0x248c, 0xffffffff, 0x00000010,
 613         0x248d, 0xffffffff, 0x000f000e,
 614         0x248e, 0xffffffff, 0x00120011,
 615         0x248f, 0xffffffff, 0x000c000b,
 616         0x2490, 0xffffffff, 0x000e000d,
 617         0x2491, 0xffffffff, 0x00000011,
 618         0x2492, 0xffffffff, 0x0010000f,
 619         0x2493, 0xffffffff, 0x00130012,
 620         0x2494, 0xffffffff, 0x000d000c,
 621         0x2495, 0xffffffff, 0x000f000e,
 622         0x2496, 0xffffffff, 0x00100013,
 623         0x2497, 0xffffffff, 0x00120011,
 624         0x2498, 0xffffffff, 0x00150014,
 625         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 626         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 627         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 628         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 629         0x000c, 0xffffffff, 0x0000001c,
 630         0x000d, 0x000f0000, 0x000f0000,
 631         0x0583, 0xffffffff, 0x00000100,
 632         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 633         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 634         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 635         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 636         0x157a, 0x00000001, 0x00000001,
 637         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 638         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 639         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 640         0x3430, 0xfffffff0, 0x00000100,
 641         0x3630, 0xfffffff0, 0x00000100,
 642 };
 643
 644 static const u32 verde_mgcg_cgcg_init[] =
 645 {
 646         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 647         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 648         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 649         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 650         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 651         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 652         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 653         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 654         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 655         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 656         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 657         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 658         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 659         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 660         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 661         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 662         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 663         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 664         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 665         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 666         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 667         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 668         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 669         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 670         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 671         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 672         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 673         0x2458, 0xffffffff, 0x00010000,
 674         0x2459, 0xffffffff, 0x00030002,
 675         0x245a, 0xffffffff, 0x00040007,
 676         0x245b, 0xffffffff, 0x00060005,
 677         0x245c, 0xffffffff, 0x00090008,
 678         0x245d, 0xffffffff, 0x00020001,
 679         0x245e, 0xffffffff, 0x00040003,
 680         0x245f, 0xffffffff, 0x00000007,
 681         0x2460, 0xffffffff, 0x00060005,
 682         0x2461, 0xffffffff, 0x00090008,
 683         0x2462, 0xffffffff, 0x00030002,
 684         0x2463, 0xffffffff, 0x00050004,
 685         0x2464, 0xffffffff, 0x00000008,
 686         0x2465, 0xffffffff, 0x00070006,
 687         0x2466, 0xffffffff, 0x000a0009,
 688         0x2467, 0xffffffff, 0x00040003,
 689         0x2468, 0xffffffff, 0x00060005,
 690         0x2469, 0xffffffff, 0x00000009,
 691         0x246a, 0xffffffff, 0x00080007,
 692         0x246b, 0xffffffff, 0x000b000a,
 693         0x246c, 0xffffffff, 0x00050004,
 694         0x246d, 0xffffffff, 0x00070006,
 695         0x246e, 0xffffffff, 0x0008000b,
 696         0x246f, 0xffffffff, 0x000a0009,
 697         0x2470, 0xffffffff, 0x000d000c,
 698         0x2480, 0xffffffff, 0x00090008,
 699         0x2481, 0xffffffff, 0x000b000a,
 700         0x2482, 0xffffffff, 0x000c000f,
 701         0x2483, 0xffffffff, 0x000e000d,
 702         0x2484, 0xffffffff, 0x00110010,
 703         0x2485, 0xffffffff, 0x000a0009,
 704         0x2486, 0xffffffff, 0x000c000b,
 705         0x2487, 0xffffffff, 0x0000000f,
 706         0x2488, 0xffffffff, 0x000e000d,
 707         0x2489, 0xffffffff, 0x00110010,
 708         0x248a, 0xffffffff, 0x000b000a,
 709         0x248b, 0xffffffff, 0x000d000c,
 710         0x248c, 0xffffffff, 0x00000010,
 711         0x248d, 0xffffffff, 0x000f000e,
 712         0x248e, 0xffffffff, 0x00120011,
 713         0x248f, 0xffffffff, 0x000c000b,
 714         0x2490, 0xffffffff, 0x000e000d,
 715         0x2491, 0xffffffff, 0x00000011,
 716         0x2492, 0xffffffff, 0x0010000f,
 717         0x2493, 0xffffffff, 0x00130012,
 718         0x2494, 0xffffffff, 0x000d000c,
 719         0x2495, 0xffffffff, 0x000f000e,
 720         0x2496, 0xffffffff, 0x00100013,
 721         0x2497, 0xffffffff, 0x00120011,
 722         0x2498, 0xffffffff, 0x00150014,
 723         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 724         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 725         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 726         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 727         0x000c, 0xffffffff, 0x0000001c,
 728         0x000d, 0x000f0000, 0x000f0000,
 729         0x0583, 0xffffffff, 0x00000100,
 730         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 731         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 732         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 733         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 734         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 735         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 736         0x157a, 0x00000001, 0x00000001,
 737         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 738         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 739         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 740         0x3430, 0xfffffff0, 0x00000100,
 741         0x3630, 0xfffffff0, 0x00000100,
 742 };
 743
 744 static const u32 oland_mgcg_cgcg_init[] =
 745 {
 746         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 747         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 748         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 749         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 750         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 751         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 752         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 753         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 754         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 755         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 756         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 757         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 758         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 759         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 760         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 761         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 762         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 763         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 764         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 765         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 766         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 767         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 768         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 769         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 770         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 771         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 772         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 773         0x2458, 0xffffffff, 0x00010000,
 774         0x2459, 0xffffffff, 0x00030002,
 775         0x245a, 0xffffffff, 0x00040007,
 776         0x245b, 0xffffffff, 0x00060005,
 777         0x245c, 0xffffffff, 0x00090008,
 778         0x245d, 0xffffffff, 0x00020001,
 779         0x245e, 0xffffffff, 0x00040003,
 780         0x245f, 0xffffffff, 0x00000007,
 781         0x2460, 0xffffffff, 0x00060005,
 782         0x2461, 0xffffffff, 0x00090008,
 783         0x2462, 0xffffffff, 0x00030002,
 784         0x2463, 0xffffffff, 0x00050004,
 785         0x2464, 0xffffffff, 0x00000008,
 786         0x2465, 0xffffffff, 0x00070006,
 787         0x2466, 0xffffffff, 0x000a0009,
 788         0x2467, 0xffffffff, 0x00040003,
 789         0x2468, 0xffffffff, 0x00060005,
 790         0x2469, 0xffffffff, 0x00000009,
 791         0x246a, 0xffffffff, 0x00080007,
 792         0x246b, 0xffffffff, 0x000b000a,
 793         0x246c, 0xffffffff, 0x00050004,
 794         0x246d, 0xffffffff, 0x00070006,
 795         0x246e, 0xffffffff, 0x0008000b,
 796         0x246f, 0xffffffff, 0x000a0009,
 797         0x2470, 0xffffffff, 0x000d000c,
 798         0x2471, 0xffffffff, 0x00060005,
 799         0x2472, 0xffffffff, 0x00080007,
 800         0x2473, 0xffffffff, 0x0000000b,
 801         0x2474, 0xffffffff, 0x000a0009,
 802         0x2475, 0xffffffff, 0x000d000c,
 803         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 804         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 805         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 806         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 807         0x000c, 0xffffffff, 0x0000001c,
 808         0x000d, 0x000f0000, 0x000f0000,
 809         0x0583, 0xffffffff, 0x00000100,
 810         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 811         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 812         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 813         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 814         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 815         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 816         0x157a, 0x00000001, 0x00000001,
 817         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 818         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 819         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 820         0x3430, 0xfffffff0, 0x00000100,
 821         0x3630, 0xfffffff0, 0x00000100,
 822 };
 823
 824 static const u32 hainan_mgcg_cgcg_init[] =
 825 {
 826         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 827         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 828         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 829         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 830         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 831         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 832         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 833         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 834         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 835         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 836         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 837         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 838         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 839         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 840         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 841         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 842         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 843         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 844         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 845         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 846         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 847         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 848         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 849         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 850         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 851         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 852         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 853         0x2458, 0xffffffff, 0x00010000,
 854         0x2459, 0xffffffff, 0x00030002,
 855         0x245a, 0xffffffff, 0x00040007,
 856         0x245b, 0xffffffff, 0x00060005,
 857         0x245c, 0xffffffff, 0x00090008,
 858         0x245d, 0xffffffff, 0x00020001,
 859         0x245e, 0xffffffff, 0x00040003,
 860         0x245f, 0xffffffff, 0x00000007,
 861         0x2460, 0xffffffff, 0x00060005,
 862         0x2461, 0xffffffff, 0x00090008,
 863         0x2462, 0xffffffff, 0x00030002,
 864         0x2463, 0xffffffff, 0x00050004,
 865         0x2464, 0xffffffff, 0x00000008,
 866         0x2465, 0xffffffff, 0x00070006,
 867         0x2466, 0xffffffff, 0x000a0009,
 868         0x2467, 0xffffffff, 0x00040003,
 869         0x2468, 0xffffffff, 0x00060005,
 870         0x2469, 0xffffffff, 0x00000009,
 871         0x246a, 0xffffffff, 0x00080007,
 872         0x246b, 0xffffffff, 0x000b000a,
 873         0x246c, 0xffffffff, 0x00050004,
 874         0x246d, 0xffffffff, 0x00070006,
 875         0x246e, 0xffffffff, 0x0008000b,
 876         0x246f, 0xffffffff, 0x000a0009,
 877         0x2470, 0xffffffff, 0x000d000c,
 878         0x2471, 0xffffffff, 0x00060005,
 879         0x2472, 0xffffffff, 0x00080007,
 880         0x2473, 0xffffffff, 0x0000000b,
 881         0x2474, 0xffffffff, 0x000a0009,
 882         0x2475, 0xffffffff, 0x000d000c,
 883         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 884         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 885         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 886         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 887         0x000c, 0xffffffff, 0x0000001c,
 888         0x000d, 0x000f0000, 0x000f0000,
 889         0x0583, 0xffffffff, 0x00000100,
 890         0x0409, 0xffffffff, 0x00000100,
 891         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 892         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 893         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 894         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 895         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 896         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 897         0x3430, 0xfffffff0, 0x00000100,
 898         0x3630, 0xfffffff0, 0x00000100,
 899 };
 900
 901 static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 902 {
 903         unsigned long flags;
 904         u32 r;
 905
 906         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 907         WREG32(AMDGPU_PCIE_INDEX, reg);
 908         (void)RREG32(AMDGPU_PCIE_INDEX);
 909         r = RREG32(AMDGPU_PCIE_DATA);
 910         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 911         return r;
 912 }
 913
 914 static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 915 {
 916         unsigned long flags;
 917
 918         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 919         WREG32(AMDGPU_PCIE_INDEX, reg);
 920         (void)RREG32(AMDGPU_PCIE_INDEX);
 921         WREG32(AMDGPU_PCIE_DATA, v);
 922         (void)RREG32(AMDGPU_PCIE_DATA);
 923         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 924 }
 925
 926 static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
 927 {
 928         unsigned long flags;
 929         u32 r;
 930
 931         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 932         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 933         (void)RREG32(PCIE_PORT_INDEX);
 934         r = RREG32(PCIE_PORT_DATA);
 935         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 936         return r;
 937 }
 938
 939 static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 940 {
 941         unsigned long flags;
 942
 943         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 944         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 945         (void)RREG32(PCIE_PORT_INDEX);
 946         WREG32(PCIE_PORT_DATA, (v));
 947         (void)RREG32(PCIE_PORT_DATA);
 948         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 949 }
 950
 951 static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
 952 {
 953         unsigned long flags;
 954         u32 r;
 955
 956         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 957         WREG32(SMC_IND_INDEX_0, (reg));
 958         r = RREG32(SMC_IND_DATA_0);
 959         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 960         return r;
 961 }
 962
 963 static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 964 {
 965         unsigned long flags;
 966
 967         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 968         WREG32(SMC_IND_INDEX_0, (reg));
 969         WREG32(SMC_IND_DATA_0, (v));
 970         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 971 }
 972
 973 static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
 974         {GRBM_STATUS},
 975         {GB_ADDR_CONFIG},
 976         {MC_ARB_RAMCFG},
 977         {GB_TILE_MODE0},
 978         {GB_TILE_MODE1},
 979         {GB_TILE_MODE2},
 980         {GB_TILE_MODE3},
 981         {GB_TILE_MODE4},
 982         {GB_TILE_MODE5},
 983         {GB_TILE_MODE6},
 984         {GB_TILE_MODE7},
 985         {GB_TILE_MODE8},
 986         {GB_TILE_MODE9},
 987         {GB_TILE_MODE10},
 988         {GB_TILE_MODE11},
 989         {GB_TILE_MODE12},
 990         {GB_TILE_MODE13},
 991         {GB_TILE_MODE14},
 992         {GB_TILE_MODE15},
 993         {GB_TILE_MODE16},
 994         {GB_TILE_MODE17},
 995         {GB_TILE_MODE18},
 996         {GB_TILE_MODE19},
 997         {GB_TILE_MODE20},
 998         {GB_TILE_MODE21},
 999         {GB_TILE_MODE22},
1000         {GB_TILE_MODE23},
1001         {GB_TILE_MODE24},
1002         {GB_TILE_MODE25},
1003         {GB_TILE_MODE26},
1004         {GB_TILE_MODE27},
1005         {GB_TILE_MODE28},
1006         {GB_TILE_MODE29},
1007         {GB_TILE_MODE30},
1008         {GB_TILE_MODE31},
1009         {CC_RB_BACKEND_DISABLE, true},
1010         {GC_USER_RB_BACKEND_DISABLE, true},
1011         {PA_SC_RASTER_CONFIG, true},
1012 };
1013
1014 static uint32_t si_get_register_value(struct amdgpu_device *adev,
1015                                       bool indexed, u32 se_num,
1016                                       u32 sh_num, u32 reg_offset)
1017 {
1018         if (indexed) {
1019                 uint32_t val;
1020                 unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1021                 unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1022
1023                 switch (reg_offset) {
1024                 case mmCC_RB_BACKEND_DISABLE:
1025                         return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1026                 case mmGC_USER_RB_BACKEND_DISABLE:
1027                         return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1028                 case mmPA_SC_RASTER_CONFIG:
1029                         return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1030                 }
1031
1032                 mutex_lock(&adev->grbm_idx_mutex);
1033                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1034                         amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1035
1036                 val = RREG32(reg_offset);
1037
1038                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1039                         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1040                 mutex_unlock(&adev->grbm_idx_mutex);
1041                 return val;
1042         } else {
1043                 unsigned idx;
1044
1045                 switch (reg_offset) {
1046                 case mmGB_ADDR_CONFIG:
1047                         return adev->gfx.config.gb_addr_config;
1048                 case mmMC_ARB_RAMCFG:
1049                         return adev->gfx.config.mc_arb_ramcfg;
1050                 case mmGB_TILE_MODE0:
1051                 case mmGB_TILE_MODE1:
1052                 case mmGB_TILE_MODE2:
1053                 case mmGB_TILE_MODE3:
1054                 case mmGB_TILE_MODE4:
1055                 case mmGB_TILE_MODE5:
1056                 case mmGB_TILE_MODE6:
1057                 case mmGB_TILE_MODE7:
1058                 case mmGB_TILE_MODE8:
1059                 case mmGB_TILE_MODE9:
1060                 case mmGB_TILE_MODE10:
1061                 case mmGB_TILE_MODE11:
1062                 case mmGB_TILE_MODE12:
1063                 case mmGB_TILE_MODE13:
1064                 case mmGB_TILE_MODE14:
1065                 case mmGB_TILE_MODE15:
1066                 case mmGB_TILE_MODE16:
1067                 case mmGB_TILE_MODE17:
1068                 case mmGB_TILE_MODE18:
1069                 case mmGB_TILE_MODE19:
1070                 case mmGB_TILE_MODE20:
1071                 case mmGB_TILE_MODE21:
1072                 case mmGB_TILE_MODE22:
1073                 case mmGB_TILE_MODE23:
1074                 case mmGB_TILE_MODE24:
1075                 case mmGB_TILE_MODE25:
1076                 case mmGB_TILE_MODE26:
1077                 case mmGB_TILE_MODE27:
1078                 case mmGB_TILE_MODE28:
1079                 case mmGB_TILE_MODE29:
1080                 case mmGB_TILE_MODE30:
1081                 case mmGB_TILE_MODE31:
1082                         idx = (reg_offset - mmGB_TILE_MODE0);
1083                         return adev->gfx.config.tile_mode_array[idx];
1084                 default:
1085                         return RREG32(reg_offset);
1086                 }
1087         }
1088 }
1089 static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1090                              u32 sh_num, u32 reg_offset, u32 *value)
1091 {
1092         uint32_t i;
1093
1094         *value = 0;
1095         for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1096                 bool indexed = si_allowed_read_registers[i].grbm_indexed;
1097
1098                 if (reg_offset != si_allowed_read_registers[i].reg_offset)
1099                         continue;
1100
1101                 *value = si_get_register_value(adev, indexed, se_num, sh_num,
1102                                                reg_offset);
1103                 return 0;
1104         }
1105         return -EINVAL;
1106 }
1107
1108 static bool si_read_disabled_bios(struct amdgpu_device *adev)
1109 {
1110         u32 bus_cntl;
1111         u32 d1vga_control = 0;
1112         u32 d2vga_control = 0;
1113         u32 vga_render_control = 0;
1114         u32 rom_cntl;
1115         bool r;
1116
1117         bus_cntl = RREG32(R600_BUS_CNTL);
1118         if (adev->mode_info.num_crtc) {
1119                 d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
1120                 d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
1121                 vga_render_control = RREG32(VGA_RENDER_CONTROL);
1122         }
1123         rom_cntl = RREG32(R600_ROM_CNTL);
1124
1125         /* enable the rom */
1126         WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
1127         if (adev->mode_info.num_crtc) {
1128                 /* Disable VGA mode */
1129                 WREG32(AVIVO_D1VGA_CONTROL,
1130                        (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1131                                           AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1132                 WREG32(AVIVO_D2VGA_CONTROL,
1133                        (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1134                                           AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1135                 WREG32(VGA_RENDER_CONTROL,
1136                        (vga_render_control & C_000300_VGA_VSTATUS_CNTL));
1137         }
1138         WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1139
1140         r = amdgpu_read_bios(adev);
1141
1142         /* restore regs */
1143         WREG32(R600_BUS_CNTL, bus_cntl);
1144         if (adev->mode_info.num_crtc) {
1145                 WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
1146                 WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
1147                 WREG32(VGA_RENDER_CONTROL, vga_render_control);
1148         }
1149         WREG32(R600_ROM_CNTL, rom_cntl);
1150         return r;
1151 }
1152
1153 #define mmROM_INDEX 0x2A
1154 #define mmROM_DATA  0x2B
1155
1156 static bool si_read_bios_from_rom(struct amdgpu_device *adev,
1157                                   u8 *bios, u32 length_bytes)
1158 {
1159         u32 *dw_ptr;
1160         u32 i, length_dw;
1161
1162         if (bios == NULL)
1163                 return false;
1164         if (length_bytes == 0)
1165                 return false;
1166         /* APU vbios image is part of sbios image */
1167         if (adev->flags & AMD_IS_APU)
1168                 return false;
1169
1170         dw_ptr = (u32 *)bios;
1171         length_dw = ALIGN(length_bytes, 4) / 4;
1172         /* set rom index to 0 */
1173         WREG32(mmROM_INDEX, 0);
1174         for (i = 0; i < length_dw; i++)
1175                 dw_ptr[i] = RREG32(mmROM_DATA);
1176
1177         return true;
1178 }
1179
1180 //xxx: not implemented
1181 static int si_asic_reset(struct amdgpu_device *adev)
1182 {
1183         return 0;
1184 }
1185
1186 static u32 si_get_config_memsize(struct amdgpu_device *adev)
1187 {
1188         return RREG32(mmCONFIG_MEMSIZE);
1189 }
1190
1191 static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1192 {
1193         uint32_t temp;
1194
1195         temp = RREG32(CONFIG_CNTL);
1196         if (state == false) {
1197                 temp &= ~(1<<0);
1198                 temp |= (1<<1);
1199         } else {
1200                 temp &= ~(1<<1);
1201         }
1202         WREG32(CONFIG_CNTL, temp);
1203 }
1204
1205 static u32 si_get_xclk(struct amdgpu_device *adev)
1206 {
1207         u32 reference_clock = adev->clock.spll.reference_freq;
1208         u32 tmp;
1209
1210         tmp = RREG32(CG_CLKPIN_CNTL_2);
1211         if (tmp & MUX_TCLK_TO_XCLK)
1212                 return TCLK;
1213
1214         tmp = RREG32(CG_CLKPIN_CNTL);
1215         if (tmp & XTALIN_DIVIDE)
1216                 return reference_clock / 4;
1217
1218         return reference_clock;
1219 }
1220
1221 //xxx:not implemented
1222 static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1223 {
1224         return 0;
1225 }
1226
1227 static void si_detect_hw_virtualization(struct amdgpu_device *adev)
1228 {
1229         if (is_virtual_machine()) /* passthrough mode */
1230                 adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
1231 }
1232
1233 static void si_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1234 {
1235         if (!ring || !ring->funcs->emit_wreg) {
1236                 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1237                 RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1238         } else {
1239                 amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1240         }
1241 }
1242
1243 static void si_invalidate_hdp(struct amdgpu_device *adev,
1244                               struct amdgpu_ring *ring)
1245 {
1246         if (!ring || !ring->funcs->emit_wreg) {
1247                 WREG32(mmHDP_DEBUG0, 1);
1248                 RREG32(mmHDP_DEBUG0);
1249         } else {
1250                 amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1251         }
1252 }
1253
1254 static const struct amdgpu_asic_funcs si_asic_funcs =
1255 {
1256         .read_disabled_bios = &si_read_disabled_bios,
1257         .read_bios_from_rom = &si_read_bios_from_rom,
1258         .read_register = &si_read_register,
1259         .reset = &si_asic_reset,
1260         .set_vga_state = &si_vga_set_state,
1261         .get_xclk = &si_get_xclk,
1262         .set_uvd_clocks = &si_set_uvd_clocks,
1263         .set_vce_clocks = NULL,
1264         .get_config_memsize = &si_get_config_memsize,
1265         .flush_hdp = &si_flush_hdp,
1266         .invalidate_hdp = &si_invalidate_hdp,
1267 };
1268
1269 static uint32_t si_get_rev_id(struct amdgpu_device *adev)
1270 {
1271         return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1272                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1273 }
1274
1275 static int si_common_early_init(void *handle)
1276 {
1277         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1278
1279         adev->smc_rreg = &si_smc_rreg;
1280         adev->smc_wreg = &si_smc_wreg;
1281         adev->pcie_rreg = &si_pcie_rreg;
1282         adev->pcie_wreg = &si_pcie_wreg;
1283         adev->pciep_rreg = &si_pciep_rreg;
1284         adev->pciep_wreg = &si_pciep_wreg;
1285         adev->uvd_ctx_rreg = NULL;
1286         adev->uvd_ctx_wreg = NULL;
1287         adev->didt_rreg = NULL;
1288         adev->didt_wreg = NULL;
1289
1290         adev->asic_funcs = &si_asic_funcs;
1291
1292         adev->rev_id = si_get_rev_id(adev);
1293         adev->external_rev_id = 0xFF;
1294         switch (adev->asic_type) {
1295         case CHIP_TAHITI:
1296                 adev->cg_flags =
1297                         AMD_CG_SUPPORT_GFX_MGCG |
1298                         AMD_CG_SUPPORT_GFX_MGLS |
1299                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1300                         AMD_CG_SUPPORT_GFX_CGLS |
1301                         AMD_CG_SUPPORT_GFX_CGTS |
1302                         AMD_CG_SUPPORT_GFX_CP_LS |
1303                         AMD_CG_SUPPORT_MC_MGCG |
1304                         AMD_CG_SUPPORT_SDMA_MGCG |
1305                         AMD_CG_SUPPORT_BIF_LS |
1306                         AMD_CG_SUPPORT_VCE_MGCG |
1307                         AMD_CG_SUPPORT_UVD_MGCG |
1308                         AMD_CG_SUPPORT_HDP_LS |
1309                         AMD_CG_SUPPORT_HDP_MGCG;
1310                         adev->pg_flags = 0;
1311                 adev->external_rev_id = (adev->rev_id == 0) ? 1 :
1312                                         (adev->rev_id == 1) ? 5 : 6;
1313                 break;
1314         case CHIP_PITCAIRN:
1315                 adev->cg_flags =
1316                         AMD_CG_SUPPORT_GFX_MGCG |
1317                         AMD_CG_SUPPORT_GFX_MGLS |
1318                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1319                         AMD_CG_SUPPORT_GFX_CGLS |
1320                         AMD_CG_SUPPORT_GFX_CGTS |
1321                         AMD_CG_SUPPORT_GFX_CP_LS |
1322                         AMD_CG_SUPPORT_GFX_RLC_LS |
1323                         AMD_CG_SUPPORT_MC_LS |
1324                         AMD_CG_SUPPORT_MC_MGCG |
1325                         AMD_CG_SUPPORT_SDMA_MGCG |
1326                         AMD_CG_SUPPORT_BIF_LS |
1327                         AMD_CG_SUPPORT_VCE_MGCG |
1328                         AMD_CG_SUPPORT_UVD_MGCG |
1329                         AMD_CG_SUPPORT_HDP_LS |
1330                         AMD_CG_SUPPORT_HDP_MGCG;
1331                 adev->pg_flags = 0;
1332                 adev->external_rev_id = adev->rev_id + 20;
1333                 break;
1334
1335         case CHIP_VERDE:
1336                 adev->cg_flags =
1337                         AMD_CG_SUPPORT_GFX_MGCG |
1338                         AMD_CG_SUPPORT_GFX_MGLS |
1339                         AMD_CG_SUPPORT_GFX_CGLS |
1340                         AMD_CG_SUPPORT_GFX_CGTS |
1341                         AMD_CG_SUPPORT_GFX_CGTS_LS |
1342                         AMD_CG_SUPPORT_GFX_CP_LS |
1343                         AMD_CG_SUPPORT_MC_LS |
1344                         AMD_CG_SUPPORT_MC_MGCG |
1345                         AMD_CG_SUPPORT_SDMA_MGCG |
1346                         AMD_CG_SUPPORT_SDMA_LS |
1347                         AMD_CG_SUPPORT_BIF_LS |
1348                         AMD_CG_SUPPORT_VCE_MGCG |
1349                         AMD_CG_SUPPORT_UVD_MGCG |
1350                         AMD_CG_SUPPORT_HDP_LS |
1351                         AMD_CG_SUPPORT_HDP_MGCG;
1352                 adev->pg_flags = 0;
1353                 //???
1354                 adev->external_rev_id = adev->rev_id + 40;
1355                 break;
1356         case CHIP_OLAND:
1357                 adev->cg_flags =
1358                         AMD_CG_SUPPORT_GFX_MGCG |
1359                         AMD_CG_SUPPORT_GFX_MGLS |
1360                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1361                         AMD_CG_SUPPORT_GFX_CGLS |
1362                         AMD_CG_SUPPORT_GFX_CGTS |
1363                         AMD_CG_SUPPORT_GFX_CP_LS |
1364                         AMD_CG_SUPPORT_GFX_RLC_LS |
1365                         AMD_CG_SUPPORT_MC_LS |
1366                         AMD_CG_SUPPORT_MC_MGCG |
1367                         AMD_CG_SUPPORT_SDMA_MGCG |
1368                         AMD_CG_SUPPORT_BIF_LS |
1369                         AMD_CG_SUPPORT_UVD_MGCG |
1370                         AMD_CG_SUPPORT_HDP_LS |
1371                         AMD_CG_SUPPORT_HDP_MGCG;
1372                 adev->pg_flags = 0;
1373                 adev->external_rev_id = 60;
1374                 break;
1375         case CHIP_HAINAN:
1376                 adev->cg_flags =
1377                         AMD_CG_SUPPORT_GFX_MGCG |
1378                         AMD_CG_SUPPORT_GFX_MGLS |
1379                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1380                         AMD_CG_SUPPORT_GFX_CGLS |
1381                         AMD_CG_SUPPORT_GFX_CGTS |
1382                         AMD_CG_SUPPORT_GFX_CP_LS |
1383                         AMD_CG_SUPPORT_GFX_RLC_LS |
1384                         AMD_CG_SUPPORT_MC_LS |
1385                         AMD_CG_SUPPORT_MC_MGCG |
1386                         AMD_CG_SUPPORT_SDMA_MGCG |
1387                         AMD_CG_SUPPORT_BIF_LS |
1388                         AMD_CG_SUPPORT_HDP_LS |
1389                         AMD_CG_SUPPORT_HDP_MGCG;
1390                 adev->pg_flags = 0;
1391                 adev->external_rev_id = 70;
1392                 break;
1393
1394         default:
1395                 return -EINVAL;
1396         }
1397
1398         return 0;
1399 }
1400
1401 static int si_common_sw_init(void *handle)
1402 {
1403         return 0;
1404 }
1405
1406 static int si_common_sw_fini(void *handle)
1407 {
1408         return 0;
1409 }
1410
1411
1412 static void si_init_golden_registers(struct amdgpu_device *adev)
1413 {
1414         switch (adev->asic_type) {
1415         case CHIP_TAHITI:
1416                 amdgpu_device_program_register_sequence(adev,
1417                                                         tahiti_golden_registers,
1418                                                         ARRAY_SIZE(tahiti_golden_registers));
1419                 amdgpu_device_program_register_sequence(adev,
1420                                                         tahiti_golden_rlc_registers,
1421                                                         ARRAY_SIZE(tahiti_golden_rlc_registers));
1422                 amdgpu_device_program_register_sequence(adev,
1423                                                         tahiti_mgcg_cgcg_init,
1424                                                         ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1425                 amdgpu_device_program_register_sequence(adev,
1426                                                         tahiti_golden_registers2,
1427                                                         ARRAY_SIZE(tahiti_golden_registers2));
1428                 break;
1429         case CHIP_PITCAIRN:
1430                 amdgpu_device_program_register_sequence(adev,
1431                                                         pitcairn_golden_registers,
1432                                                         ARRAY_SIZE(pitcairn_golden_registers));
1433                 amdgpu_device_program_register_sequence(adev,
1434                                                         pitcairn_golden_rlc_registers,
1435                                                         ARRAY_SIZE(pitcairn_golden_rlc_registers));
1436                 amdgpu_device_program_register_sequence(adev,
1437                                                         pitcairn_mgcg_cgcg_init,
1438                                                         ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1439                 break;
1440         case CHIP_VERDE:
1441                 amdgpu_device_program_register_sequence(adev,
1442                                                         verde_golden_registers,
1443                                                         ARRAY_SIZE(verde_golden_registers));
1444                 amdgpu_device_program_register_sequence(adev,
1445                                                         verde_golden_rlc_registers,
1446                                                         ARRAY_SIZE(verde_golden_rlc_registers));
1447                 amdgpu_device_program_register_sequence(adev,
1448                                                         verde_mgcg_cgcg_init,
1449                                                         ARRAY_SIZE(verde_mgcg_cgcg_init));
1450                 amdgpu_device_program_register_sequence(adev,
1451                                                         verde_pg_init,
1452                                                         ARRAY_SIZE(verde_pg_init));
1453                 break;
1454         case CHIP_OLAND:
1455                 amdgpu_device_program_register_sequence(adev,
1456                                                         oland_golden_registers,
1457                                                         ARRAY_SIZE(oland_golden_registers));
1458                 amdgpu_device_program_register_sequence(adev,
1459                                                         oland_golden_rlc_registers,
1460                                                         ARRAY_SIZE(oland_golden_rlc_registers));
1461                 amdgpu_device_program_register_sequence(adev,
1462                                                         oland_mgcg_cgcg_init,
1463                                                         ARRAY_SIZE(oland_mgcg_cgcg_init));
1464                 break;
1465         case CHIP_HAINAN:
1466                 amdgpu_device_program_register_sequence(adev,
1467                                                         hainan_golden_registers,
1468                                                         ARRAY_SIZE(hainan_golden_registers));
1469                 amdgpu_device_program_register_sequence(adev,
1470                                                         hainan_golden_registers2,
1471                                                         ARRAY_SIZE(hainan_golden_registers2));
1472                 amdgpu_device_program_register_sequence(adev,
1473                                                         hainan_mgcg_cgcg_init,
1474                                                         ARRAY_SIZE(hainan_mgcg_cgcg_init));
1475                 break;
1476
1477
1478         default:
1479                 BUG();
1480         }
1481 }
1482
1483 static void si_pcie_gen3_enable(struct amdgpu_device *adev)
1484 {
1485         struct pci_dev *root = adev->pdev->bus->self;
1486         int bridge_pos, gpu_pos;
1487         u32 speed_cntl, mask, current_data_rate;
1488         int ret, i;
1489         u16 tmp16;
1490
1491         if (pci_is_root_bus(adev->pdev->bus))
1492                 return;
1493
1494         if (amdgpu_pcie_gen2 == 0)
1495                 return;
1496
1497         if (adev->flags & AMD_IS_APU)
1498                 return;
1499
1500         ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
1501         if (ret != 0)
1502                 return;
1503
1504         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
1505                 return;
1506
1507         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1508         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
1509                 LC_CURRENT_DATA_RATE_SHIFT;
1510         if (mask & DRM_PCIE_SPEED_80) {
1511                 if (current_data_rate == 2) {
1512                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1513                         return;
1514                 }
1515                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1516         } else if (mask & DRM_PCIE_SPEED_50) {
1517                 if (current_data_rate == 1) {
1518                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1519                         return;
1520                 }
1521                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1522         }
1523
1524         bridge_pos = pci_pcie_cap(root);
1525         if (!bridge_pos)
1526                 return;
1527
1528         gpu_pos = pci_pcie_cap(adev->pdev);
1529         if (!gpu_pos)
1530                 return;
1531
1532         if (mask & DRM_PCIE_SPEED_80) {
1533                 if (current_data_rate != 2) {
1534                         u16 bridge_cfg, gpu_cfg;
1535                         u16 bridge_cfg2, gpu_cfg2;
1536                         u32 max_lw, current_lw, tmp;
1537
1538                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1539                         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1540
1541                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1542                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1543
1544                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1545                         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1546
1547                         tmp = RREG32_PCIE(PCIE_LC_STATUS1);
1548                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
1549                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
1550
1551                         if (current_lw < max_lw) {
1552                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1553                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
1554                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
1555                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
1556                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
1557                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
1558                                 }
1559                         }
1560
1561                         for (i = 0; i < 10; i++) {
1562                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1563                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1564                                         break;
1565
1566                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1567                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1568
1569                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1570                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1571
1572                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1573                                 tmp |= LC_SET_QUIESCE;
1574                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1575
1576                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1577                                 tmp |= LC_REDO_EQ;
1578                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1579
1580                                 mdelay(100);
1581
1582                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1583                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1584                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1585                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1586
1587                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1588                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1589                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1590                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1591
1592                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1593                                 tmp16 &= ~((1 << 4) | (7 << 9));
1594                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1595                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1596
1597                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1598                                 tmp16 &= ~((1 << 4) | (7 << 9));
1599                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1600                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1601
1602                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1603                                 tmp &= ~LC_SET_QUIESCE;
1604                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1605                         }
1606                 }
1607         }
1608
1609         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
1610         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
1611         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1612
1613         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1614         tmp16 &= ~0xf;
1615         if (mask & DRM_PCIE_SPEED_80)
1616                 tmp16 |= 3;
1617         else if (mask & DRM_PCIE_SPEED_50)
1618                 tmp16 |= 2;
1619         else
1620                 tmp16 |= 1;
1621         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1622
1623         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1624         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
1625         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1626
1627         for (i = 0; i < adev->usec_timeout; i++) {
1628                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1629                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
1630                         break;
1631                 udelay(1);
1632         }
1633 }
1634
1635 static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
1636 {
1637         unsigned long flags;
1638         u32 r;
1639
1640         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1641         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1642         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
1643         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1644         return r;
1645 }
1646
1647 static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1648 {
1649         unsigned long flags;
1650
1651         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1652         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1653         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
1654         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1655 }
1656
1657 static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
1658 {
1659         unsigned long flags;
1660         u32 r;
1661
1662         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1663         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1664         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
1665         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1666         return r;
1667 }
1668
1669 static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1670 {
1671         unsigned long flags;
1672
1673         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1674         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1675         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
1676         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1677 }
1678 static void si_program_aspm(struct amdgpu_device *adev)
1679 {
1680         u32 data, orig;
1681         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1682         bool disable_clkreq = false;
1683
1684         if (amdgpu_aspm == 0)
1685                 return;
1686
1687         if (adev->flags & AMD_IS_APU)
1688                 return;
1689         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
1690         data &= ~LC_XMIT_N_FTS_MASK;
1691         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
1692         if (orig != data)
1693                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
1694
1695         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
1696         data |= LC_GO_TO_RECOVERY;
1697         if (orig != data)
1698                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
1699
1700         orig = data = RREG32_PCIE(PCIE_P_CNTL);
1701         data |= P_IGNORE_EDB_ERR;
1702         if (orig != data)
1703                 WREG32_PCIE(PCIE_P_CNTL, data);
1704
1705         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
1706         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
1707         data |= LC_PMI_TO_L1_DIS;
1708         if (!disable_l0s)
1709                 data |= LC_L0S_INACTIVITY(7);
1710
1711         if (!disable_l1) {
1712                 data |= LC_L1_INACTIVITY(7);
1713                 data &= ~LC_PMI_TO_L1_DIS;
1714                 if (orig != data)
1715                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1716
1717                 if (!disable_plloff_in_l1) {
1718                         bool clk_req_support;
1719
1720                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1721                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1722                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1723                         if (orig != data)
1724                                 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1725
1726                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1727                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1728                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1729                         if (orig != data)
1730                                 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1731
1732                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1733                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1734                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1735                         if (orig != data)
1736                                 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1737
1738                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1739                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1740                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1741                         if (orig != data)
1742                                 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1743
1744                         if ((adev->family != CHIP_OLAND) && (adev->family != CHIP_HAINAN)) {
1745                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1746                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
1747                                 if (orig != data)
1748                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1749
1750                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1751                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
1752                                 if (orig != data)
1753                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1754
1755                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);
1756                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
1757                                 if (orig != data)
1758                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);
1759
1760                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);
1761                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
1762                                 if (orig != data)
1763                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);
1764
1765                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1766                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
1767                                 if (orig != data)
1768                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1769
1770                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1771                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
1772                                 if (orig != data)
1773                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1774
1775                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);
1776                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
1777                                 if (orig != data)
1778                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);
1779
1780                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);
1781                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
1782                                 if (orig != data)
1783                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);
1784                         }
1785                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1786                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
1787                         data |= LC_DYN_LANES_PWR_STATE(3);
1788                         if (orig != data)
1789                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
1790
1791                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);
1792                         data &= ~LS2_EXIT_TIME_MASK;
1793                         if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
1794                                 data |= LS2_EXIT_TIME(5);
1795                         if (orig != data)
1796                                 si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);
1797
1798                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);
1799                         data &= ~LS2_EXIT_TIME_MASK;
1800                         if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
1801                                 data |= LS2_EXIT_TIME(5);
1802                         if (orig != data)
1803                                 si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);
1804
1805                         if (!disable_clkreq &&
1806                             !pci_is_root_bus(adev->pdev->bus)) {
1807                                 struct pci_dev *root = adev->pdev->bus->self;
1808                                 u32 lnkcap;
1809
1810                                 clk_req_support = false;
1811                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1812                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1813                                         clk_req_support = true;
1814                         } else {
1815                                 clk_req_support = false;
1816                         }
1817
1818                         if (clk_req_support) {
1819                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
1820                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
1821                                 if (orig != data)
1822                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
1823
1824                                 orig = data = RREG32(THM_CLK_CNTL);
1825                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
1826                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
1827                                 if (orig != data)
1828                                         WREG32(THM_CLK_CNTL, data);
1829
1830                                 orig = data = RREG32(MISC_CLK_CNTL);
1831                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
1832                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
1833                                 if (orig != data)
1834                                         WREG32(MISC_CLK_CNTL, data);
1835
1836                                 orig = data = RREG32(CG_CLKPIN_CNTL);
1837                                 data &= ~BCLK_AS_XCLK;
1838                                 if (orig != data)
1839                                         WREG32(CG_CLKPIN_CNTL, data);
1840
1841                                 orig = data = RREG32(CG_CLKPIN_CNTL_2);
1842                                 data &= ~FORCE_BIF_REFCLK_EN;
1843                                 if (orig != data)
1844                                         WREG32(CG_CLKPIN_CNTL_2, data);
1845
1846                                 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
1847                                 data &= ~MPLL_CLKOUT_SEL_MASK;
1848                                 data |= MPLL_CLKOUT_SEL(4);
1849                                 if (orig != data)
1850                                         WREG32(MPLL_BYPASSCLK_SEL, data);
1851
1852                                 orig = data = RREG32(SPLL_CNTL_MODE);
1853                                 data &= ~SPLL_REFCLK_SEL_MASK;
1854                                 if (orig != data)
1855                                         WREG32(SPLL_CNTL_MODE, data);
1856                         }
1857                 }
1858         } else {
1859                 if (orig != data)
1860                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1861         }
1862
1863         orig = data = RREG32_PCIE(PCIE_CNTL2);
1864         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
1865         if (orig != data)
1866                 WREG32_PCIE(PCIE_CNTL2, data);
1867
1868         if (!disable_l0s) {
1869                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
1870                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
1871                         data = RREG32_PCIE(PCIE_LC_STATUS1);
1872                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
1873                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
1874                                 data &= ~LC_L0S_INACTIVITY_MASK;
1875                                 if (orig != data)
1876                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1877                         }
1878                 }
1879         }
1880 }
1881
1882 static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
1883 {
1884         int readrq;
1885         u16 v;
1886
1887         readrq = pcie_get_readrq(adev->pdev);
1888         v = ffs(readrq) - 8;
1889         if ((v == 0) || (v == 6) || (v == 7))
1890                 pcie_set_readrq(adev->pdev, 512);
1891 }
1892
1893 static int si_common_hw_init(void *handle)
1894 {
1895         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1896
1897         si_fix_pci_max_read_req_size(adev);
1898         si_init_golden_registers(adev);
1899         si_pcie_gen3_enable(adev);
1900         si_program_aspm(adev);
1901
1902         return 0;
1903 }
1904
1905 static int si_common_hw_fini(void *handle)
1906 {
1907         return 0;
1908 }
1909
1910 static int si_common_suspend(void *handle)
1911 {
1912         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1913
1914         return si_common_hw_fini(adev);
1915 }
1916
1917 static int si_common_resume(void *handle)
1918 {
1919         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1920
1921         return si_common_hw_init(adev);
1922 }
1923
1924 static bool si_common_is_idle(void *handle)
1925 {
1926         return true;
1927 }
1928
1929 static int si_common_wait_for_idle(void *handle)
1930 {
1931         return 0;
1932 }
1933
1934 static int si_common_soft_reset(void *handle)
1935 {
1936         return 0;
1937 }
1938
1939 static int si_common_set_clockgating_state(void *handle,
1940                                             enum amd_clockgating_state state)
1941 {
1942         return 0;
1943 }
1944
1945 static int si_common_set_powergating_state(void *handle,
1946                                             enum amd_powergating_state state)
1947 {
1948         return 0;
1949 }
1950
1951 static const struct amd_ip_funcs si_common_ip_funcs = {
1952         .name = "si_common",
1953         .early_init = si_common_early_init,
1954         .late_init = NULL,
1955         .sw_init = si_common_sw_init,
1956         .sw_fini = si_common_sw_fini,
1957         .hw_init = si_common_hw_init,
1958         .hw_fini = si_common_hw_fini,
1959         .suspend = si_common_suspend,
1960         .resume = si_common_resume,
1961         .is_idle = si_common_is_idle,
1962         .wait_for_idle = si_common_wait_for_idle,
1963         .soft_reset = si_common_soft_reset,
1964         .set_clockgating_state = si_common_set_clockgating_state,
1965         .set_powergating_state = si_common_set_powergating_state,
1966 };
1967
1968 static const struct amdgpu_ip_block_version si_common_ip_block =
1969 {
1970         .type = AMD_IP_BLOCK_TYPE_COMMON,
1971         .major = 1,
1972         .minor = 0,
1973         .rev = 0,
1974         .funcs = &si_common_ip_funcs,
1975 };
1976
1977 int si_set_ip_blocks(struct amdgpu_device *adev)
1978 {
1979         si_detect_hw_virtualization(adev);
1980
1981         switch (adev->asic_type) {
1982         case CHIP_VERDE:
1983         case CHIP_TAHITI:
1984         case CHIP_PITCAIRN:
1985                 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
1986                 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
1987                 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
1988                 amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
1989                 if (adev->enable_virtual_display)
1990                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
1991                 else
1992                         amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);
1993                 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
1994                 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
1995                 /* amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); */
1996                 /* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
1997                 break;
1998         case CHIP_OLAND:
1999                 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2000                 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2001                 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2002                 amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
2003                 if (adev->enable_virtual_display)
2004                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2005                 else
2006                         amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);
2007                 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2008                 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2009                 /* amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); */
2010                 /* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2011                 break;
2012         case CHIP_HAINAN:
2013                 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2014                 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2015                 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2016                 amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
2017                 if (adev->enable_virtual_display)
2018                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2019                 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2020                 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2021                 break;
2022         default:
2023                 BUG();
2024         }
2025         return 0;
2026 }
2027