drivers/gpu/drm/amd/amdgpu/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_atombios.h"
  30 #include "amdgpu_ih.h"
  31 #include "amdgpu_uvd.h"
  32 #include "amdgpu_vce.h"
  33 #include "cikd.h"
  34 #include "atom.h"
  35 #include "amd_pcie.h"
  36
  37 #include "cik.h"
  38 #include "gmc_v7_0.h"
  39 #include "cik_ih.h"
  40 #include "dce_v8_0.h"
  41 #include "gfx_v7_0.h"
  42 #include "cik_sdma.h"
  43 #include "uvd_v4_2.h"
  44 #include "vce_v2_0.h"
  45 #include "cik_dpm.h"
  46
  47 #include "uvd/uvd_4_2_d.h"
  48
  49 #include "smu/smu_7_0_1_d.h"
  50 #include "smu/smu_7_0_1_sh_mask.h"
  51
  52 #include "dce/dce_8_0_d.h"
  53 #include "dce/dce_8_0_sh_mask.h"
  54
  55 #include "bif/bif_4_1_d.h"
  56 #include "bif/bif_4_1_sh_mask.h"
  57
  58 #include "gca/gfx_7_2_d.h"
  59 #include "gca/gfx_7_2_enum.h"
  60 #include "gca/gfx_7_2_sh_mask.h"
  61
  62 #include "gmc/gmc_7_1_d.h"
  63 #include "gmc/gmc_7_1_sh_mask.h"
  64
  65 #include "oss/oss_2_0_d.h"
  66 #include "oss/oss_2_0_sh_mask.h"
  67
  68 #include "amdgpu_dm.h"
  69 #include "amdgpu_amdkfd.h"
  70 #include "amdgpu_powerplay.h"
  71 #include "dce_virtual.h"
  72
  73 /*
  74  * Indirect registers accessor
  75  */
  76 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
  77 {
  78         unsigned long flags;
  79         u32 r;
  80
  81         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  82         WREG32(mmPCIE_INDEX, reg);
  83         (void)RREG32(mmPCIE_INDEX);
  84         r = RREG32(mmPCIE_DATA);
  85         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  86         return r;
  87 }
  88
  89 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  90 {
  91         unsigned long flags;
  92
  93         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  94         WREG32(mmPCIE_INDEX, reg);
  95         (void)RREG32(mmPCIE_INDEX);
  96         WREG32(mmPCIE_DATA, v);
  97         (void)RREG32(mmPCIE_DATA);
  98         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  99 }
 100
 101 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 102 {
 103         unsigned long flags;
 104         u32 r;
 105
 106         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 107         WREG32(mmSMC_IND_INDEX_0, (reg));
 108         r = RREG32(mmSMC_IND_DATA_0);
 109         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 110         return r;
 111 }
 112
 113 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 114 {
 115         unsigned long flags;
 116
 117         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 118         WREG32(mmSMC_IND_INDEX_0, (reg));
 119         WREG32(mmSMC_IND_DATA_0, (v));
 120         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 121 }
 122
 123 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 124 {
 125         unsigned long flags;
 126         u32 r;
 127
 128         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 129         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 130         r = RREG32(mmUVD_CTX_DATA);
 131         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 132         return r;
 133 }
 134
 135 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 136 {
 137         unsigned long flags;
 138
 139         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 140         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 141         WREG32(mmUVD_CTX_DATA, (v));
 142         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 143 }
 144
 145 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 146 {
 147         unsigned long flags;
 148         u32 r;
 149
 150         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 151         WREG32(mmDIDT_IND_INDEX, (reg));
 152         r = RREG32(mmDIDT_IND_DATA);
 153         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 154         return r;
 155 }
 156
 157 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 158 {
 159         unsigned long flags;
 160
 161         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 162         WREG32(mmDIDT_IND_INDEX, (reg));
 163         WREG32(mmDIDT_IND_DATA, (v));
 164         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 165 }
 166
 167 static const u32 bonaire_golden_spm_registers[] =
 168 {
 169         0xc200, 0xe0ffffff, 0xe0000000
 170 };
 171
 172 static const u32 bonaire_golden_common_registers[] =
 173 {
 174         0x31dc, 0xffffffff, 0x00000800,
 175         0x31dd, 0xffffffff, 0x00000800,
 176         0x31e6, 0xffffffff, 0x00007fbf,
 177         0x31e7, 0xffffffff, 0x00007faf
 178 };
 179
 180 static const u32 bonaire_golden_registers[] =
 181 {
 182         0xcd5, 0x00000333, 0x00000333,
 183         0xcd4, 0x000c0fc0, 0x00040200,
 184         0x2684, 0x00010000, 0x00058208,
 185         0xf000, 0xffff1fff, 0x00140000,
 186         0xf080, 0xfdfc0fff, 0x00000100,
 187         0xf08d, 0x40000000, 0x40000200,
 188         0x260c, 0xffffffff, 0x00000000,
 189         0x260d, 0xf00fffff, 0x00000400,
 190         0x260e, 0x0002021c, 0x00020200,
 191         0x31e, 0x00000080, 0x00000000,
 192         0x16ec, 0x000000f0, 0x00000070,
 193         0x16f0, 0xf0311fff, 0x80300000,
 194         0x263e, 0x73773777, 0x12010001,
 195         0xd43, 0x00810000, 0x408af000,
 196         0x1c0c, 0x31000111, 0x00000011,
 197         0xbd2, 0x73773777, 0x12010001,
 198         0x883, 0x00007fb6, 0x0021a1b1,
 199         0x884, 0x00007fb6, 0x002021b1,
 200         0x860, 0x00007fb6, 0x00002191,
 201         0x886, 0x00007fb6, 0x002121b1,
 202         0x887, 0x00007fb6, 0x002021b1,
 203         0x877, 0x00007fb6, 0x00002191,
 204         0x878, 0x00007fb6, 0x00002191,
 205         0xd8a, 0x0000003f, 0x0000000a,
 206         0xd8b, 0x0000003f, 0x0000000a,
 207         0xab9, 0x00073ffe, 0x000022a2,
 208         0x903, 0x000007ff, 0x00000000,
 209         0x2285, 0xf000003f, 0x00000007,
 210         0x22fc, 0x00002001, 0x00000001,
 211         0x22c9, 0xffffffff, 0x00ffffff,
 212         0xc281, 0x0000ff0f, 0x00000000,
 213         0xa293, 0x07ffffff, 0x06000000,
 214         0x136, 0x00000fff, 0x00000100,
 215         0xf9e, 0x00000001, 0x00000002,
 216         0x2440, 0x03000000, 0x0362c688,
 217         0x2300, 0x000000ff, 0x00000001,
 218         0x390, 0x00001fff, 0x00001fff,
 219         0x2418, 0x0000007f, 0x00000020,
 220         0x2542, 0x00010000, 0x00010000,
 221         0x2b05, 0x000003ff, 0x000000f3,
 222         0x2b03, 0xffffffff, 0x00001032
 223 };
 224
 225 static const u32 bonaire_mgcg_cgcg_init[] =
 226 {
 227         0x3108, 0xffffffff, 0xfffffffc,
 228         0xc200, 0xffffffff, 0xe0000000,
 229         0xf0a8, 0xffffffff, 0x00000100,
 230         0xf082, 0xffffffff, 0x00000100,
 231         0xf0b0, 0xffffffff, 0xc0000100,
 232         0xf0b2, 0xffffffff, 0xc0000100,
 233         0xf0b1, 0xffffffff, 0xc0000100,
 234         0x1579, 0xffffffff, 0x00600100,
 235         0xf0a0, 0xffffffff, 0x00000100,
 236         0xf085, 0xffffffff, 0x06000100,
 237         0xf088, 0xffffffff, 0x00000100,
 238         0xf086, 0xffffffff, 0x06000100,
 239         0xf081, 0xffffffff, 0x00000100,
 240         0xf0b8, 0xffffffff, 0x00000100,
 241         0xf089, 0xffffffff, 0x00000100,
 242         0xf080, 0xffffffff, 0x00000100,
 243         0xf08c, 0xffffffff, 0x00000100,
 244         0xf08d, 0xffffffff, 0x00000100,
 245         0xf094, 0xffffffff, 0x00000100,
 246         0xf095, 0xffffffff, 0x00000100,
 247         0xf096, 0xffffffff, 0x00000100,
 248         0xf097, 0xffffffff, 0x00000100,
 249         0xf098, 0xffffffff, 0x00000100,
 250         0xf09f, 0xffffffff, 0x00000100,
 251         0xf09e, 0xffffffff, 0x00000100,
 252         0xf084, 0xffffffff, 0x06000100,
 253         0xf0a4, 0xffffffff, 0x00000100,
 254         0xf09d, 0xffffffff, 0x00000100,
 255         0xf0ad, 0xffffffff, 0x00000100,
 256         0xf0ac, 0xffffffff, 0x00000100,
 257         0xf09c, 0xffffffff, 0x00000100,
 258         0xc200, 0xffffffff, 0xe0000000,
 259         0xf008, 0xffffffff, 0x00010000,
 260         0xf009, 0xffffffff, 0x00030002,
 261         0xf00a, 0xffffffff, 0x00040007,
 262         0xf00b, 0xffffffff, 0x00060005,
 263         0xf00c, 0xffffffff, 0x00090008,
 264         0xf00d, 0xffffffff, 0x00010000,
 265         0xf00e, 0xffffffff, 0x00030002,
 266         0xf00f, 0xffffffff, 0x00040007,
 267         0xf010, 0xffffffff, 0x00060005,
 268         0xf011, 0xffffffff, 0x00090008,
 269         0xf012, 0xffffffff, 0x00010000,
 270         0xf013, 0xffffffff, 0x00030002,
 271         0xf014, 0xffffffff, 0x00040007,
 272         0xf015, 0xffffffff, 0x00060005,
 273         0xf016, 0xffffffff, 0x00090008,
 274         0xf017, 0xffffffff, 0x00010000,
 275         0xf018, 0xffffffff, 0x00030002,
 276         0xf019, 0xffffffff, 0x00040007,
 277         0xf01a, 0xffffffff, 0x00060005,
 278         0xf01b, 0xffffffff, 0x00090008,
 279         0xf01c, 0xffffffff, 0x00010000,
 280         0xf01d, 0xffffffff, 0x00030002,
 281         0xf01e, 0xffffffff, 0x00040007,
 282         0xf01f, 0xffffffff, 0x00060005,
 283         0xf020, 0xffffffff, 0x00090008,
 284         0xf021, 0xffffffff, 0x00010000,
 285         0xf022, 0xffffffff, 0x00030002,
 286         0xf023, 0xffffffff, 0x00040007,
 287         0xf024, 0xffffffff, 0x00060005,
 288         0xf025, 0xffffffff, 0x00090008,
 289         0xf026, 0xffffffff, 0x00010000,
 290         0xf027, 0xffffffff, 0x00030002,
 291         0xf028, 0xffffffff, 0x00040007,
 292         0xf029, 0xffffffff, 0x00060005,
 293         0xf02a, 0xffffffff, 0x00090008,
 294         0xf000, 0xffffffff, 0x96e00200,
 295         0x21c2, 0xffffffff, 0x00900100,
 296         0x3109, 0xffffffff, 0x0020003f,
 297         0xe, 0xffffffff, 0x0140001c,
 298         0xf, 0x000f0000, 0x000f0000,
 299         0x88, 0xffffffff, 0xc060000c,
 300         0x89, 0xc0000fff, 0x00000100,
 301         0x3e4, 0xffffffff, 0x00000100,
 302         0x3e6, 0x00000101, 0x00000000,
 303         0x82a, 0xffffffff, 0x00000104,
 304         0x1579, 0xff000fff, 0x00000100,
 305         0xc33, 0xc0000fff, 0x00000104,
 306         0x3079, 0x00000001, 0x00000001,
 307         0x3403, 0xff000ff0, 0x00000100,
 308         0x3603, 0xff000ff0, 0x00000100
 309 };
 310
 311 static const u32 spectre_golden_spm_registers[] =
 312 {
 313         0xc200, 0xe0ffffff, 0xe0000000
 314 };
 315
 316 static const u32 spectre_golden_common_registers[] =
 317 {
 318         0x31dc, 0xffffffff, 0x00000800,
 319         0x31dd, 0xffffffff, 0x00000800,
 320         0x31e6, 0xffffffff, 0x00007fbf,
 321         0x31e7, 0xffffffff, 0x00007faf
 322 };
 323
 324 static const u32 spectre_golden_registers[] =
 325 {
 326         0xf000, 0xffff1fff, 0x96940200,
 327         0xf003, 0xffff0001, 0xff000000,
 328         0xf080, 0xfffc0fff, 0x00000100,
 329         0x1bb6, 0x00010101, 0x00010000,
 330         0x260d, 0xf00fffff, 0x00000400,
 331         0x260e, 0xfffffffc, 0x00020200,
 332         0x16ec, 0x000000f0, 0x00000070,
 333         0x16f0, 0xf0311fff, 0x80300000,
 334         0x263e, 0x73773777, 0x12010001,
 335         0x26df, 0x00ff0000, 0x00fc0000,
 336         0xbd2, 0x73773777, 0x12010001,
 337         0x2285, 0xf000003f, 0x00000007,
 338         0x22c9, 0xffffffff, 0x00ffffff,
 339         0xa0d4, 0x3f3f3fff, 0x00000082,
 340         0xa0d5, 0x0000003f, 0x00000000,
 341         0xf9e, 0x00000001, 0x00000002,
 342         0x244f, 0xffff03df, 0x00000004,
 343         0x31da, 0x00000008, 0x00000008,
 344         0x2300, 0x000008ff, 0x00000800,
 345         0x2542, 0x00010000, 0x00010000,
 346         0x2b03, 0xffffffff, 0x54763210,
 347         0x853e, 0x01ff01ff, 0x00000002,
 348         0x8526, 0x007ff800, 0x00200000,
 349         0x8057, 0xffffffff, 0x00000f40,
 350         0xc24d, 0xffffffff, 0x00000001
 351 };
 352
 353 static const u32 spectre_mgcg_cgcg_init[] =
 354 {
 355         0x3108, 0xffffffff, 0xfffffffc,
 356         0xc200, 0xffffffff, 0xe0000000,
 357         0xf0a8, 0xffffffff, 0x00000100,
 358         0xf082, 0xffffffff, 0x00000100,
 359         0xf0b0, 0xffffffff, 0x00000100,
 360         0xf0b2, 0xffffffff, 0x00000100,
 361         0xf0b1, 0xffffffff, 0x00000100,
 362         0x1579, 0xffffffff, 0x00600100,
 363         0xf0a0, 0xffffffff, 0x00000100,
 364         0xf085, 0xffffffff, 0x06000100,
 365         0xf088, 0xffffffff, 0x00000100,
 366         0xf086, 0xffffffff, 0x06000100,
 367         0xf081, 0xffffffff, 0x00000100,
 368         0xf0b8, 0xffffffff, 0x00000100,
 369         0xf089, 0xffffffff, 0x00000100,
 370         0xf080, 0xffffffff, 0x00000100,
 371         0xf08c, 0xffffffff, 0x00000100,
 372         0xf08d, 0xffffffff, 0x00000100,
 373         0xf094, 0xffffffff, 0x00000100,
 374         0xf095, 0xffffffff, 0x00000100,
 375         0xf096, 0xffffffff, 0x00000100,
 376         0xf097, 0xffffffff, 0x00000100,
 377         0xf098, 0xffffffff, 0x00000100,
 378         0xf09f, 0xffffffff, 0x00000100,
 379         0xf09e, 0xffffffff, 0x00000100,
 380         0xf084, 0xffffffff, 0x06000100,
 381         0xf0a4, 0xffffffff, 0x00000100,
 382         0xf09d, 0xffffffff, 0x00000100,
 383         0xf0ad, 0xffffffff, 0x00000100,
 384         0xf0ac, 0xffffffff, 0x00000100,
 385         0xf09c, 0xffffffff, 0x00000100,
 386         0xc200, 0xffffffff, 0xe0000000,
 387         0xf008, 0xffffffff, 0x00010000,
 388         0xf009, 0xffffffff, 0x00030002,
 389         0xf00a, 0xffffffff, 0x00040007,
 390         0xf00b, 0xffffffff, 0x00060005,
 391         0xf00c, 0xffffffff, 0x00090008,
 392         0xf00d, 0xffffffff, 0x00010000,
 393         0xf00e, 0xffffffff, 0x00030002,
 394         0xf00f, 0xffffffff, 0x00040007,
 395         0xf010, 0xffffffff, 0x00060005,
 396         0xf011, 0xffffffff, 0x00090008,
 397         0xf012, 0xffffffff, 0x00010000,
 398         0xf013, 0xffffffff, 0x00030002,
 399         0xf014, 0xffffffff, 0x00040007,
 400         0xf015, 0xffffffff, 0x00060005,
 401         0xf016, 0xffffffff, 0x00090008,
 402         0xf017, 0xffffffff, 0x00010000,
 403         0xf018, 0xffffffff, 0x00030002,
 404         0xf019, 0xffffffff, 0x00040007,
 405         0xf01a, 0xffffffff, 0x00060005,
 406         0xf01b, 0xffffffff, 0x00090008,
 407         0xf01c, 0xffffffff, 0x00010000,
 408         0xf01d, 0xffffffff, 0x00030002,
 409         0xf01e, 0xffffffff, 0x00040007,
 410         0xf01f, 0xffffffff, 0x00060005,
 411         0xf020, 0xffffffff, 0x00090008,
 412         0xf021, 0xffffffff, 0x00010000,
 413         0xf022, 0xffffffff, 0x00030002,
 414         0xf023, 0xffffffff, 0x00040007,
 415         0xf024, 0xffffffff, 0x00060005,
 416         0xf025, 0xffffffff, 0x00090008,
 417         0xf026, 0xffffffff, 0x00010000,
 418         0xf027, 0xffffffff, 0x00030002,
 419         0xf028, 0xffffffff, 0x00040007,
 420         0xf029, 0xffffffff, 0x00060005,
 421         0xf02a, 0xffffffff, 0x00090008,
 422         0xf02b, 0xffffffff, 0x00010000,
 423         0xf02c, 0xffffffff, 0x00030002,
 424         0xf02d, 0xffffffff, 0x00040007,
 425         0xf02e, 0xffffffff, 0x00060005,
 426         0xf02f, 0xffffffff, 0x00090008,
 427         0xf000, 0xffffffff, 0x96e00200,
 428         0x21c2, 0xffffffff, 0x00900100,
 429         0x3109, 0xffffffff, 0x0020003f,
 430         0xe, 0xffffffff, 0x0140001c,
 431         0xf, 0x000f0000, 0x000f0000,
 432         0x88, 0xffffffff, 0xc060000c,
 433         0x89, 0xc0000fff, 0x00000100,
 434         0x3e4, 0xffffffff, 0x00000100,
 435         0x3e6, 0x00000101, 0x00000000,
 436         0x82a, 0xffffffff, 0x00000104,
 437         0x1579, 0xff000fff, 0x00000100,
 438         0xc33, 0xc0000fff, 0x00000104,
 439         0x3079, 0x00000001, 0x00000001,
 440         0x3403, 0xff000ff0, 0x00000100,
 441         0x3603, 0xff000ff0, 0x00000100
 442 };
 443
 444 static const u32 kalindi_golden_spm_registers[] =
 445 {
 446         0xc200, 0xe0ffffff, 0xe0000000
 447 };
 448
 449 static const u32 kalindi_golden_common_registers[] =
 450 {
 451         0x31dc, 0xffffffff, 0x00000800,
 452         0x31dd, 0xffffffff, 0x00000800,
 453         0x31e6, 0xffffffff, 0x00007fbf,
 454         0x31e7, 0xffffffff, 0x00007faf
 455 };
 456
 457 static const u32 kalindi_golden_registers[] =
 458 {
 459         0xf000, 0xffffdfff, 0x6e944040,
 460         0x1579, 0xff607fff, 0xfc000100,
 461         0xf088, 0xff000fff, 0x00000100,
 462         0xf089, 0xff000fff, 0x00000100,
 463         0xf080, 0xfffc0fff, 0x00000100,
 464         0x1bb6, 0x00010101, 0x00010000,
 465         0x260c, 0xffffffff, 0x00000000,
 466         0x260d, 0xf00fffff, 0x00000400,
 467         0x16ec, 0x000000f0, 0x00000070,
 468         0x16f0, 0xf0311fff, 0x80300000,
 469         0x263e, 0x73773777, 0x12010001,
 470         0x263f, 0xffffffff, 0x00000010,
 471         0x26df, 0x00ff0000, 0x00fc0000,
 472         0x200c, 0x00001f0f, 0x0000100a,
 473         0xbd2, 0x73773777, 0x12010001,
 474         0x902, 0x000fffff, 0x000c007f,
 475         0x2285, 0xf000003f, 0x00000007,
 476         0x22c9, 0x3fff3fff, 0x00ffcfff,
 477         0xc281, 0x0000ff0f, 0x00000000,
 478         0xa293, 0x07ffffff, 0x06000000,
 479         0x136, 0x00000fff, 0x00000100,
 480         0xf9e, 0x00000001, 0x00000002,
 481         0x31da, 0x00000008, 0x00000008,
 482         0x2300, 0x000000ff, 0x00000003,
 483         0x853e, 0x01ff01ff, 0x00000002,
 484         0x8526, 0x007ff800, 0x00200000,
 485         0x8057, 0xffffffff, 0x00000f40,
 486         0x2231, 0x001f3ae3, 0x00000082,
 487         0x2235, 0x0000001f, 0x00000010,
 488         0xc24d, 0xffffffff, 0x00000000
 489 };
 490
 491 static const u32 kalindi_mgcg_cgcg_init[] =
 492 {
 493         0x3108, 0xffffffff, 0xfffffffc,
 494         0xc200, 0xffffffff, 0xe0000000,
 495         0xf0a8, 0xffffffff, 0x00000100,
 496         0xf082, 0xffffffff, 0x00000100,
 497         0xf0b0, 0xffffffff, 0x00000100,
 498         0xf0b2, 0xffffffff, 0x00000100,
 499         0xf0b1, 0xffffffff, 0x00000100,
 500         0x1579, 0xffffffff, 0x00600100,
 501         0xf0a0, 0xffffffff, 0x00000100,
 502         0xf085, 0xffffffff, 0x06000100,
 503         0xf088, 0xffffffff, 0x00000100,
 504         0xf086, 0xffffffff, 0x06000100,
 505         0xf081, 0xffffffff, 0x00000100,
 506         0xf0b8, 0xffffffff, 0x00000100,
 507         0xf089, 0xffffffff, 0x00000100,
 508         0xf080, 0xffffffff, 0x00000100,
 509         0xf08c, 0xffffffff, 0x00000100,
 510         0xf08d, 0xffffffff, 0x00000100,
 511         0xf094, 0xffffffff, 0x00000100,
 512         0xf095, 0xffffffff, 0x00000100,
 513         0xf096, 0xffffffff, 0x00000100,
 514         0xf097, 0xffffffff, 0x00000100,
 515         0xf098, 0xffffffff, 0x00000100,
 516         0xf09f, 0xffffffff, 0x00000100,
 517         0xf09e, 0xffffffff, 0x00000100,
 518         0xf084, 0xffffffff, 0x06000100,
 519         0xf0a4, 0xffffffff, 0x00000100,
 520         0xf09d, 0xffffffff, 0x00000100,
 521         0xf0ad, 0xffffffff, 0x00000100,
 522         0xf0ac, 0xffffffff, 0x00000100,
 523         0xf09c, 0xffffffff, 0x00000100,
 524         0xc200, 0xffffffff, 0xe0000000,
 525         0xf008, 0xffffffff, 0x00010000,
 526         0xf009, 0xffffffff, 0x00030002,
 527         0xf00a, 0xffffffff, 0x00040007,
 528         0xf00b, 0xffffffff, 0x00060005,
 529         0xf00c, 0xffffffff, 0x00090008,
 530         0xf00d, 0xffffffff, 0x00010000,
 531         0xf00e, 0xffffffff, 0x00030002,
 532         0xf00f, 0xffffffff, 0x00040007,
 533         0xf010, 0xffffffff, 0x00060005,
 534         0xf011, 0xffffffff, 0x00090008,
 535         0xf000, 0xffffffff, 0x96e00200,
 536         0x21c2, 0xffffffff, 0x00900100,
 537         0x3109, 0xffffffff, 0x0020003f,
 538         0xe, 0xffffffff, 0x0140001c,
 539         0xf, 0x000f0000, 0x000f0000,
 540         0x88, 0xffffffff, 0xc060000c,
 541         0x89, 0xc0000fff, 0x00000100,
 542         0x82a, 0xffffffff, 0x00000104,
 543         0x1579, 0xff000fff, 0x00000100,
 544         0xc33, 0xc0000fff, 0x00000104,
 545         0x3079, 0x00000001, 0x00000001,
 546         0x3403, 0xff000ff0, 0x00000100,
 547         0x3603, 0xff000ff0, 0x00000100
 548 };
 549
 550 static const u32 hawaii_golden_spm_registers[] =
 551 {
 552         0xc200, 0xe0ffffff, 0xe0000000
 553 };
 554
 555 static const u32 hawaii_golden_common_registers[] =
 556 {
 557         0xc200, 0xffffffff, 0xe0000000,
 558         0xa0d4, 0xffffffff, 0x3a00161a,
 559         0xa0d5, 0xffffffff, 0x0000002e,
 560         0x2684, 0xffffffff, 0x00018208,
 561         0x263e, 0xffffffff, 0x12011003
 562 };
 563
 564 static const u32 hawaii_golden_registers[] =
 565 {
 566         0xcd5, 0x00000333, 0x00000333,
 567         0x2684, 0x00010000, 0x00058208,
 568         0x260c, 0xffffffff, 0x00000000,
 569         0x260d, 0xf00fffff, 0x00000400,
 570         0x260e, 0x0002021c, 0x00020200,
 571         0x31e, 0x00000080, 0x00000000,
 572         0x16ec, 0x000000f0, 0x00000070,
 573         0x16f0, 0xf0311fff, 0x80300000,
 574         0xd43, 0x00810000, 0x408af000,
 575         0x1c0c, 0x31000111, 0x00000011,
 576         0xbd2, 0x73773777, 0x12010001,
 577         0x848, 0x0000007f, 0x0000001b,
 578         0x877, 0x00007fb6, 0x00002191,
 579         0xd8a, 0x0000003f, 0x0000000a,
 580         0xd8b, 0x0000003f, 0x0000000a,
 581         0xab9, 0x00073ffe, 0x000022a2,
 582         0x903, 0x000007ff, 0x00000000,
 583         0x22fc, 0x00002001, 0x00000001,
 584         0x22c9, 0xffffffff, 0x00ffffff,
 585         0xc281, 0x0000ff0f, 0x00000000,
 586         0xa293, 0x07ffffff, 0x06000000,
 587         0xf9e, 0x00000001, 0x00000002,
 588         0x31da, 0x00000008, 0x00000008,
 589         0x31dc, 0x00000f00, 0x00000800,
 590         0x31dd, 0x00000f00, 0x00000800,
 591         0x31e6, 0x00ffffff, 0x00ff7fbf,
 592         0x31e7, 0x00ffffff, 0x00ff7faf,
 593         0x2300, 0x000000ff, 0x00000800,
 594         0x390, 0x00001fff, 0x00001fff,
 595         0x2418, 0x0000007f, 0x00000020,
 596         0x2542, 0x00010000, 0x00010000,
 597         0x2b80, 0x00100000, 0x000ff07c,
 598         0x2b05, 0x000003ff, 0x0000000f,
 599         0x2b04, 0xffffffff, 0x7564fdec,
 600         0x2b03, 0xffffffff, 0x3120b9a8,
 601         0x2b02, 0x20000000, 0x0f9c0000
 602 };
 603
 604 static const u32 hawaii_mgcg_cgcg_init[] =
 605 {
 606         0x3108, 0xffffffff, 0xfffffffd,
 607         0xc200, 0xffffffff, 0xe0000000,
 608         0xf0a8, 0xffffffff, 0x00000100,
 609         0xf082, 0xffffffff, 0x00000100,
 610         0xf0b0, 0xffffffff, 0x00000100,
 611         0xf0b2, 0xffffffff, 0x00000100,
 612         0xf0b1, 0xffffffff, 0x00000100,
 613         0x1579, 0xffffffff, 0x00200100,
 614         0xf0a0, 0xffffffff, 0x00000100,
 615         0xf085, 0xffffffff, 0x06000100,
 616         0xf088, 0xffffffff, 0x00000100,
 617         0xf086, 0xffffffff, 0x06000100,
 618         0xf081, 0xffffffff, 0x00000100,
 619         0xf0b8, 0xffffffff, 0x00000100,
 620         0xf089, 0xffffffff, 0x00000100,
 621         0xf080, 0xffffffff, 0x00000100,
 622         0xf08c, 0xffffffff, 0x00000100,
 623         0xf08d, 0xffffffff, 0x00000100,
 624         0xf094, 0xffffffff, 0x00000100,
 625         0xf095, 0xffffffff, 0x00000100,
 626         0xf096, 0xffffffff, 0x00000100,
 627         0xf097, 0xffffffff, 0x00000100,
 628         0xf098, 0xffffffff, 0x00000100,
 629         0xf09f, 0xffffffff, 0x00000100,
 630         0xf09e, 0xffffffff, 0x00000100,
 631         0xf084, 0xffffffff, 0x06000100,
 632         0xf0a4, 0xffffffff, 0x00000100,
 633         0xf09d, 0xffffffff, 0x00000100,
 634         0xf0ad, 0xffffffff, 0x00000100,
 635         0xf0ac, 0xffffffff, 0x00000100,
 636         0xf09c, 0xffffffff, 0x00000100,
 637         0xc200, 0xffffffff, 0xe0000000,
 638         0xf008, 0xffffffff, 0x00010000,
 639         0xf009, 0xffffffff, 0x00030002,
 640         0xf00a, 0xffffffff, 0x00040007,
 641         0xf00b, 0xffffffff, 0x00060005,
 642         0xf00c, 0xffffffff, 0x00090008,
 643         0xf00d, 0xffffffff, 0x00010000,
 644         0xf00e, 0xffffffff, 0x00030002,
 645         0xf00f, 0xffffffff, 0x00040007,
 646         0xf010, 0xffffffff, 0x00060005,
 647         0xf011, 0xffffffff, 0x00090008,
 648         0xf012, 0xffffffff, 0x00010000,
 649         0xf013, 0xffffffff, 0x00030002,
 650         0xf014, 0xffffffff, 0x00040007,
 651         0xf015, 0xffffffff, 0x00060005,
 652         0xf016, 0xffffffff, 0x00090008,
 653         0xf017, 0xffffffff, 0x00010000,
 654         0xf018, 0xffffffff, 0x00030002,
 655         0xf019, 0xffffffff, 0x00040007,
 656         0xf01a, 0xffffffff, 0x00060005,
 657         0xf01b, 0xffffffff, 0x00090008,
 658         0xf01c, 0xffffffff, 0x00010000,
 659         0xf01d, 0xffffffff, 0x00030002,
 660         0xf01e, 0xffffffff, 0x00040007,
 661         0xf01f, 0xffffffff, 0x00060005,
 662         0xf020, 0xffffffff, 0x00090008,
 663         0xf021, 0xffffffff, 0x00010000,
 664         0xf022, 0xffffffff, 0x00030002,
 665         0xf023, 0xffffffff, 0x00040007,
 666         0xf024, 0xffffffff, 0x00060005,
 667         0xf025, 0xffffffff, 0x00090008,
 668         0xf026, 0xffffffff, 0x00010000,
 669         0xf027, 0xffffffff, 0x00030002,
 670         0xf028, 0xffffffff, 0x00040007,
 671         0xf029, 0xffffffff, 0x00060005,
 672         0xf02a, 0xffffffff, 0x00090008,
 673         0xf02b, 0xffffffff, 0x00010000,
 674         0xf02c, 0xffffffff, 0x00030002,
 675         0xf02d, 0xffffffff, 0x00040007,
 676         0xf02e, 0xffffffff, 0x00060005,
 677         0xf02f, 0xffffffff, 0x00090008,
 678         0xf030, 0xffffffff, 0x00010000,
 679         0xf031, 0xffffffff, 0x00030002,
 680         0xf032, 0xffffffff, 0x00040007,
 681         0xf033, 0xffffffff, 0x00060005,
 682         0xf034, 0xffffffff, 0x00090008,
 683         0xf035, 0xffffffff, 0x00010000,
 684         0xf036, 0xffffffff, 0x00030002,
 685         0xf037, 0xffffffff, 0x00040007,
 686         0xf038, 0xffffffff, 0x00060005,
 687         0xf039, 0xffffffff, 0x00090008,
 688         0xf03a, 0xffffffff, 0x00010000,
 689         0xf03b, 0xffffffff, 0x00030002,
 690         0xf03c, 0xffffffff, 0x00040007,
 691         0xf03d, 0xffffffff, 0x00060005,
 692         0xf03e, 0xffffffff, 0x00090008,
 693         0x30c6, 0xffffffff, 0x00020200,
 694         0xcd4, 0xffffffff, 0x00000200,
 695         0x570, 0xffffffff, 0x00000400,
 696         0x157a, 0xffffffff, 0x00000000,
 697         0xbd4, 0xffffffff, 0x00000902,
 698         0xf000, 0xffffffff, 0x96940200,
 699         0x21c2, 0xffffffff, 0x00900100,
 700         0x3109, 0xffffffff, 0x0020003f,
 701         0xe, 0xffffffff, 0x0140001c,
 702         0xf, 0x000f0000, 0x000f0000,
 703         0x88, 0xffffffff, 0xc060000c,
 704         0x89, 0xc0000fff, 0x00000100,
 705         0x3e4, 0xffffffff, 0x00000100,
 706         0x3e6, 0x00000101, 0x00000000,
 707         0x82a, 0xffffffff, 0x00000104,
 708         0x1579, 0xff000fff, 0x00000100,
 709         0xc33, 0xc0000fff, 0x00000104,
 710         0x3079, 0x00000001, 0x00000001,
 711         0x3403, 0xff000ff0, 0x00000100,
 712         0x3603, 0xff000ff0, 0x00000100
 713 };
 714
 715 static const u32 godavari_golden_registers[] =
 716 {
 717         0x1579, 0xff607fff, 0xfc000100,
 718         0x1bb6, 0x00010101, 0x00010000,
 719         0x260c, 0xffffffff, 0x00000000,
 720         0x260c0, 0xf00fffff, 0x00000400,
 721         0x184c, 0xffffffff, 0x00010000,
 722         0x16ec, 0x000000f0, 0x00000070,
 723         0x16f0, 0xf0311fff, 0x80300000,
 724         0x263e, 0x73773777, 0x12010001,
 725         0x263f, 0xffffffff, 0x00000010,
 726         0x200c, 0x00001f0f, 0x0000100a,
 727         0xbd2, 0x73773777, 0x12010001,
 728         0x902, 0x000fffff, 0x000c007f,
 729         0x2285, 0xf000003f, 0x00000007,
 730         0x22c9, 0xffffffff, 0x00ff0fff,
 731         0xc281, 0x0000ff0f, 0x00000000,
 732         0xa293, 0x07ffffff, 0x06000000,
 733         0x136, 0x00000fff, 0x00000100,
 734         0x3405, 0x00010000, 0x00810001,
 735         0x3605, 0x00010000, 0x00810001,
 736         0xf9e, 0x00000001, 0x00000002,
 737         0x31da, 0x00000008, 0x00000008,
 738         0x31dc, 0x00000f00, 0x00000800,
 739         0x31dd, 0x00000f00, 0x00000800,
 740         0x31e6, 0x00ffffff, 0x00ff7fbf,
 741         0x31e7, 0x00ffffff, 0x00ff7faf,
 742         0x2300, 0x000000ff, 0x00000001,
 743         0x853e, 0x01ff01ff, 0x00000002,
 744         0x8526, 0x007ff800, 0x00200000,
 745         0x8057, 0xffffffff, 0x00000f40,
 746         0x2231, 0x001f3ae3, 0x00000082,
 747         0x2235, 0x0000001f, 0x00000010,
 748         0xc24d, 0xffffffff, 0x00000000
 749 };
 750
 751 static void cik_init_golden_registers(struct amdgpu_device *adev)
 752 {
 753         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
 754         mutex_lock(&adev->grbm_idx_mutex);
 755
 756         switch (adev->asic_type) {
 757         case CHIP_BONAIRE:
 758                 amdgpu_program_register_sequence(adev,
 759                                                  bonaire_mgcg_cgcg_init,
 760                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 761                 amdgpu_program_register_sequence(adev,
 762                                                  bonaire_golden_registers,
 763                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
 764                 amdgpu_program_register_sequence(adev,
 765                                                  bonaire_golden_common_registers,
 766                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
 767                 amdgpu_program_register_sequence(adev,
 768                                                  bonaire_golden_spm_registers,
 769                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
 770                 break;
 771         case CHIP_KABINI:
 772                 amdgpu_program_register_sequence(adev,
 773                                                  kalindi_mgcg_cgcg_init,
 774                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 775                 amdgpu_program_register_sequence(adev,
 776                                                  kalindi_golden_registers,
 777                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
 778                 amdgpu_program_register_sequence(adev,
 779                                                  kalindi_golden_common_registers,
 780                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 781                 amdgpu_program_register_sequence(adev,
 782                                                  kalindi_golden_spm_registers,
 783                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 784                 break;
 785         case CHIP_MULLINS:
 786                 amdgpu_program_register_sequence(adev,
 787                                                  kalindi_mgcg_cgcg_init,
 788                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 789                 amdgpu_program_register_sequence(adev,
 790                                                  godavari_golden_registers,
 791                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
 792                 amdgpu_program_register_sequence(adev,
 793                                                  kalindi_golden_common_registers,
 794                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 795                 amdgpu_program_register_sequence(adev,
 796                                                  kalindi_golden_spm_registers,
 797                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 798                 break;
 799         case CHIP_KAVERI:
 800                 amdgpu_program_register_sequence(adev,
 801                                                  spectre_mgcg_cgcg_init,
 802                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
 803                 amdgpu_program_register_sequence(adev,
 804                                                  spectre_golden_registers,
 805                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
 806                 amdgpu_program_register_sequence(adev,
 807                                                  spectre_golden_common_registers,
 808                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
 809                 amdgpu_program_register_sequence(adev,
 810                                                  spectre_golden_spm_registers,
 811                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
 812                 break;
 813         case CHIP_HAWAII:
 814                 amdgpu_program_register_sequence(adev,
 815                                                  hawaii_mgcg_cgcg_init,
 816                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 817                 amdgpu_program_register_sequence(adev,
 818                                                  hawaii_golden_registers,
 819                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
 820                 amdgpu_program_register_sequence(adev,
 821                                                  hawaii_golden_common_registers,
 822                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
 823                 amdgpu_program_register_sequence(adev,
 824                                                  hawaii_golden_spm_registers,
 825                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
 826                 break;
 827         default:
 828                 break;
 829         }
 830         mutex_unlock(&adev->grbm_idx_mutex);
 831 }
 832
 833 /**
 834  * cik_get_xclk - get the xclk
 835  *
 836  * @adev: amdgpu_device pointer
 837  *
 838  * Returns the reference clock used by the gfx engine
 839  * (CIK).
 840  */
 841 static u32 cik_get_xclk(struct amdgpu_device *adev)
 842 {
 843         u32 reference_clock = adev->clock.spll.reference_freq;
 844
 845         if (adev->flags & AMD_IS_APU) {
 846                 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 847                         return reference_clock / 2;
 848         } else {
 849                 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 850                         return reference_clock / 4;
 851         }
 852         return reference_clock;
 853 }
 854
 855 /**
 856  * cik_srbm_select - select specific register instances
 857  *
 858  * @adev: amdgpu_device pointer
 859  * @me: selected ME (micro engine)
 860  * @pipe: pipe
 861  * @queue: queue
 862  * @vmid: VMID
 863  *
 864  * Switches the currently active registers instances.  Some
 865  * registers are instanced per VMID, others are instanced per
 866  * me/pipe/queue combination.
 867  */
 868 void cik_srbm_select(struct amdgpu_device *adev,
 869                      u32 me, u32 pipe, u32 queue, u32 vmid)
 870 {
 871         u32 srbm_gfx_cntl =
 872                 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 873                 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 874                 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 875                 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 876         WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 877 }
 878
 879 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 880 {
 881         uint32_t tmp;
 882
 883         tmp = RREG32(mmCONFIG_CNTL);
 884         if (!state)
 885                 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 886         else
 887                 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 888         WREG32(mmCONFIG_CNTL, tmp);
 889 }
 890
 891 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 892 {
 893         u32 bus_cntl;
 894         u32 d1vga_control = 0;
 895         u32 d2vga_control = 0;
 896         u32 vga_render_control = 0;
 897         u32 rom_cntl;
 898         bool r;
 899
 900         bus_cntl = RREG32(mmBUS_CNTL);
 901         if (adev->mode_info.num_crtc) {
 902                 d1vga_control = RREG32(mmD1VGA_CONTROL);
 903                 d2vga_control = RREG32(mmD2VGA_CONTROL);
 904                 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 905         }
 906         rom_cntl = RREG32_SMC(ixROM_CNTL);
 907
 908         /* enable the rom */
 909         WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 910         if (adev->mode_info.num_crtc) {
 911                 /* Disable VGA mode */
 912                 WREG32(mmD1VGA_CONTROL,
 913                        (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 914                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 915                 WREG32(mmD2VGA_CONTROL,
 916                        (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 917                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 918                 WREG32(mmVGA_RENDER_CONTROL,
 919                        (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 920         }
 921         WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 922
 923         r = amdgpu_read_bios(adev);
 924
 925         /* restore regs */
 926         WREG32(mmBUS_CNTL, bus_cntl);
 927         if (adev->mode_info.num_crtc) {
 928                 WREG32(mmD1VGA_CONTROL, d1vga_control);
 929                 WREG32(mmD2VGA_CONTROL, d2vga_control);
 930                 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 931         }
 932         WREG32_SMC(ixROM_CNTL, rom_cntl);
 933         return r;
 934 }
 935
 936 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 937                                    u8 *bios, u32 length_bytes)
 938 {
 939         u32 *dw_ptr;
 940         unsigned long flags;
 941         u32 i, length_dw;
 942
 943         if (bios == NULL)
 944                 return false;
 945         if (length_bytes == 0)
 946                 return false;
 947         /* APU vbios image is part of sbios image */
 948         if (adev->flags & AMD_IS_APU)
 949                 return false;
 950
 951         dw_ptr = (u32 *)bios;
 952         length_dw = ALIGN(length_bytes, 4) / 4;
 953         /* take the smc lock since we are using the smc index */
 954         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 955         /* set rom index to 0 */
 956         WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
 957         WREG32(mmSMC_IND_DATA_0, 0);
 958         /* set index to data for continous read */
 959         WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
 960         for (i = 0; i < length_dw; i++)
 961                 dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
 962         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 963
 964         return true;
 965 }
 966
 967 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 968         {mmGRBM_STATUS},
 969         {mmGB_ADDR_CONFIG},
 970         {mmMC_ARB_RAMCFG},
 971         {mmGB_TILE_MODE0},
 972         {mmGB_TILE_MODE1},
 973         {mmGB_TILE_MODE2},
 974         {mmGB_TILE_MODE3},
 975         {mmGB_TILE_MODE4},
 976         {mmGB_TILE_MODE5},
 977         {mmGB_TILE_MODE6},
 978         {mmGB_TILE_MODE7},
 979         {mmGB_TILE_MODE8},
 980         {mmGB_TILE_MODE9},
 981         {mmGB_TILE_MODE10},
 982         {mmGB_TILE_MODE11},
 983         {mmGB_TILE_MODE12},
 984         {mmGB_TILE_MODE13},
 985         {mmGB_TILE_MODE14},
 986         {mmGB_TILE_MODE15},
 987         {mmGB_TILE_MODE16},
 988         {mmGB_TILE_MODE17},
 989         {mmGB_TILE_MODE18},
 990         {mmGB_TILE_MODE19},
 991         {mmGB_TILE_MODE20},
 992         {mmGB_TILE_MODE21},
 993         {mmGB_TILE_MODE22},
 994         {mmGB_TILE_MODE23},
 995         {mmGB_TILE_MODE24},
 996         {mmGB_TILE_MODE25},
 997         {mmGB_TILE_MODE26},
 998         {mmGB_TILE_MODE27},
 999         {mmGB_TILE_MODE28},
1000         {mmGB_TILE_MODE29},
1001         {mmGB_TILE_MODE30},
1002         {mmGB_TILE_MODE31},
1003         {mmGB_MACROTILE_MODE0},
1004         {mmGB_MACROTILE_MODE1},
1005         {mmGB_MACROTILE_MODE2},
1006         {mmGB_MACROTILE_MODE3},
1007         {mmGB_MACROTILE_MODE4},
1008         {mmGB_MACROTILE_MODE5},
1009         {mmGB_MACROTILE_MODE6},
1010         {mmGB_MACROTILE_MODE7},
1011         {mmGB_MACROTILE_MODE8},
1012         {mmGB_MACROTILE_MODE9},
1013         {mmGB_MACROTILE_MODE10},
1014         {mmGB_MACROTILE_MODE11},
1015         {mmGB_MACROTILE_MODE12},
1016         {mmGB_MACROTILE_MODE13},
1017         {mmGB_MACROTILE_MODE14},
1018         {mmGB_MACROTILE_MODE15},
1019         {mmCC_RB_BACKEND_DISABLE, true},
1020         {mmGC_USER_RB_BACKEND_DISABLE, true},
1021         {mmGB_BACKEND_MAP, false},
1022         {mmPA_SC_RASTER_CONFIG, true},
1023         {mmPA_SC_RASTER_CONFIG_1, true},
1024 };
1025
1026 static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
1027                                           u32 se_num, u32 sh_num,
1028                                           u32 reg_offset)
1029 {
1030         uint32_t val;
1031
1032         mutex_lock(&adev->grbm_idx_mutex);
1033         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1034                 amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1035
1036         val = RREG32(reg_offset);
1037
1038         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1039                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1040         mutex_unlock(&adev->grbm_idx_mutex);
1041         return val;
1042 }
1043
1044 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1045                              u32 sh_num, u32 reg_offset, u32 *value)
1046 {
1047         uint32_t i;
1048
1049         *value = 0;
1050         for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1051                 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1052                         continue;
1053
1054                 *value = cik_allowed_read_registers[i].grbm_indexed ?
1055                          cik_read_indexed_register(adev, se_num,
1056                                                    sh_num, reg_offset) :
1057                          RREG32(reg_offset);
1058                 return 0;
1059         }
1060         return -EINVAL;
1061 }
1062
1063 struct kv_reset_save_regs {
1064         u32 gmcon_reng_execute;
1065         u32 gmcon_misc;
1066         u32 gmcon_misc3;
1067 };
1068
1069 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1070                                    struct kv_reset_save_regs *save)
1071 {
1072         save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1073         save->gmcon_misc = RREG32(mmGMCON_MISC);
1074         save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1075
1076         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1077                 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1078         WREG32(mmGMCON_MISC, save->gmcon_misc &
1079                 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1080                         GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1081 }
1082
1083 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1084                                       struct kv_reset_save_regs *save)
1085 {
1086         int i;
1087
1088         WREG32(mmGMCON_PGFSM_WRITE, 0);
1089         WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1090
1091         for (i = 0; i < 5; i++)
1092                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1093
1094         WREG32(mmGMCON_PGFSM_WRITE, 0);
1095         WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1096
1097         for (i = 0; i < 5; i++)
1098                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1099
1100         WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1101         WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1102
1103         for (i = 0; i < 5; i++)
1104                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1105
1106         WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1107         WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1108
1109         for (i = 0; i < 5; i++)
1110                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1111
1112         WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1113         WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1114
1115         for (i = 0; i < 5; i++)
1116                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1117
1118         WREG32(mmGMCON_PGFSM_WRITE, 0);
1119         WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1120
1121         for (i = 0; i < 5; i++)
1122                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1123
1124         WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1125         WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1126
1127         for (i = 0; i < 5; i++)
1128                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1129
1130         WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1131         WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1132
1133         for (i = 0; i < 5; i++)
1134                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1135
1136         WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1137         WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1138
1139         for (i = 0; i < 5; i++)
1140                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1141
1142         WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1143         WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1144
1145         for (i = 0; i < 5; i++)
1146                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1147
1148         WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1149         WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1150
1151         WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1152         WREG32(mmGMCON_MISC, save->gmcon_misc);
1153         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1154 }
1155
1156 static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1157 {
1158         struct kv_reset_save_regs kv_save = { 0 };
1159         u32 i;
1160         int r = -EINVAL;
1161
1162         dev_info(adev->dev, "GPU pci config reset\n");
1163
1164         if (adev->flags & AMD_IS_APU)
1165                 kv_save_regs_for_reset(adev, &kv_save);
1166
1167         /* disable BM */
1168         pci_clear_master(adev->pdev);
1169         /* reset */
1170         amdgpu_pci_config_reset(adev);
1171
1172         udelay(100);
1173
1174         /* wait for asic to come out of reset */
1175         for (i = 0; i < adev->usec_timeout; i++) {
1176                 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1177                         /* enable BM */
1178                         pci_set_master(adev->pdev);
1179                         adev->has_hw_reset = true;
1180                         r = 0;
1181                         break;
1182                 }
1183                 udelay(1);
1184         }
1185
1186         /* does asic init need to be run first??? */
1187         if (adev->flags & AMD_IS_APU)
1188                 kv_restore_regs_for_reset(adev, &kv_save);
1189
1190         return r;
1191 }
1192
1193 /**
1194  * cik_asic_reset - soft reset GPU
1195  *
1196  * @adev: amdgpu_device pointer
1197  *
1198  * Look up which blocks are hung and attempt
1199  * to reset them.
1200  * Returns 0 for success.
1201  */
1202 static int cik_asic_reset(struct amdgpu_device *adev)
1203 {
1204         int r;
1205
1206         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1207
1208         r = cik_gpu_pci_config_reset(adev);
1209
1210         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1211
1212         return r;
1213 }
1214
1215 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1216 {
1217         return RREG32(mmCONFIG_MEMSIZE);
1218 }
1219
1220 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1221                               u32 cntl_reg, u32 status_reg)
1222 {
1223         int r, i;
1224         struct atom_clock_dividers dividers;
1225         uint32_t tmp;
1226
1227         r = amdgpu_atombios_get_clock_dividers(adev,
1228                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1229                                                clock, false, &dividers);
1230         if (r)
1231                 return r;
1232
1233         tmp = RREG32_SMC(cntl_reg);
1234         tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1235                 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1236         tmp |= dividers.post_divider;
1237         WREG32_SMC(cntl_reg, tmp);
1238
1239         for (i = 0; i < 100; i++) {
1240                 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1241                         break;
1242                 mdelay(10);
1243         }
1244         if (i == 100)
1245                 return -ETIMEDOUT;
1246
1247         return 0;
1248 }
1249
1250 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1251 {
1252         int r = 0;
1253
1254         r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1255         if (r)
1256                 return r;
1257
1258         r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1259         return r;
1260 }
1261
1262 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1263 {
1264         int r, i;
1265         struct atom_clock_dividers dividers;
1266         u32 tmp;
1267
1268         r = amdgpu_atombios_get_clock_dividers(adev,
1269                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1270                                                ecclk, false, &dividers);
1271         if (r)
1272                 return r;
1273
1274         for (i = 0; i < 100; i++) {
1275                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1276                         break;
1277                 mdelay(10);
1278         }
1279         if (i == 100)
1280                 return -ETIMEDOUT;
1281
1282         tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1283         tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1284                 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1285         tmp |= dividers.post_divider;
1286         WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1287
1288         for (i = 0; i < 100; i++) {
1289                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1290                         break;
1291                 mdelay(10);
1292         }
1293         if (i == 100)
1294                 return -ETIMEDOUT;
1295
1296         return 0;
1297 }
1298
1299 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1300 {
1301         struct pci_dev *root = adev->pdev->bus->self;
1302         int bridge_pos, gpu_pos;
1303         u32 speed_cntl, current_data_rate;
1304         int i;
1305         u16 tmp16;
1306
1307         if (pci_is_root_bus(adev->pdev->bus))
1308                 return;
1309
1310         if (amdgpu_pcie_gen2 == 0)
1311                 return;
1312
1313         if (adev->flags & AMD_IS_APU)
1314                 return;
1315
1316         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1317                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1318                 return;
1319
1320         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1321         current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1322                 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1323         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1324                 if (current_data_rate == 2) {
1325                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1326                         return;
1327                 }
1328                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1329         } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1330                 if (current_data_rate == 1) {
1331                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1332                         return;
1333                 }
1334                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1335         }
1336
1337         bridge_pos = pci_pcie_cap(root);
1338         if (!bridge_pos)
1339                 return;
1340
1341         gpu_pos = pci_pcie_cap(adev->pdev);
1342         if (!gpu_pos)
1343                 return;
1344
1345         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1346                 /* re-try equalization if gen3 is not already enabled */
1347                 if (current_data_rate != 2) {
1348                         u16 bridge_cfg, gpu_cfg;
1349                         u16 bridge_cfg2, gpu_cfg2;
1350                         u32 max_lw, current_lw, tmp;
1351
1352                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1353                         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1354
1355                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1356                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1357
1358                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1359                         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1360
1361                         tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1362                         max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1363                                 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1364                         current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1365                                 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1366
1367                         if (current_lw < max_lw) {
1368                                 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1369                                 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1370                                         tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1371                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1372                                         tmp |= (max_lw <<
1373                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1374                                         tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1375                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1376                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1377                                         WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1378                                 }
1379                         }
1380
1381                         for (i = 0; i < 10; i++) {
1382                                 /* check status */
1383                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1384                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1385                                         break;
1386
1387                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1388                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1389
1390                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1391                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1392
1393                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1394                                 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1395                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1396
1397                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1398                                 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1399                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1400
1401                                 mdelay(100);
1402
1403                                 /* linkctl */
1404                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1405                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1406                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1407                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1408
1409                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1410                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1411                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1412                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1413
1414                                 /* linkctl2 */
1415                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1416                                 tmp16 &= ~((1 << 4) | (7 << 9));
1417                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1418                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1419
1420                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1421                                 tmp16 &= ~((1 << 4) | (7 << 9));
1422                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1423                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1424
1425                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1426                                 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1427                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1428                         }
1429                 }
1430         }
1431
1432         /* set the link speed */
1433         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1434                 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1435         speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1436         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1437
1438         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1439         tmp16 &= ~0xf;
1440         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1441                 tmp16 |= 3; /* gen3 */
1442         else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1443                 tmp16 |= 2; /* gen2 */
1444         else
1445                 tmp16 |= 1; /* gen1 */
1446         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1447
1448         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1449         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1450         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1451
1452         for (i = 0; i < adev->usec_timeout; i++) {
1453                 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1454                 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1455                         break;
1456                 udelay(1);
1457         }
1458 }
1459
1460 static void cik_program_aspm(struct amdgpu_device *adev)
1461 {
1462         u32 data, orig;
1463         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1464         bool disable_clkreq = false;
1465
1466         if (amdgpu_aspm == 0)
1467                 return;
1468
1469         if (pci_is_root_bus(adev->pdev->bus))
1470                 return;
1471
1472         /* XXX double check APUs */
1473         if (adev->flags & AMD_IS_APU)
1474                 return;
1475
1476         orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1477         data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1478         data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1479                 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1480         if (orig != data)
1481                 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1482
1483         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1484         data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1485         if (orig != data)
1486                 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1487
1488         orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1489         data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1490         if (orig != data)
1491                 WREG32_PCIE(ixPCIE_P_CNTL, data);
1492
1493         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1494         data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1495                 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1496         data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1497         if (!disable_l0s)
1498                 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1499
1500         if (!disable_l1) {
1501                 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1502                 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1503                 if (orig != data)
1504                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1505
1506                 if (!disable_plloff_in_l1) {
1507                         bool clk_req_support;
1508
1509                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1510                         data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1511                                 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1512                         data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1513                                 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1514                         if (orig != data)
1515                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1516
1517                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1518                         data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1519                                 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1520                         data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1521                                 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1522                         if (orig != data)
1523                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1524
1525                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1526                         data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1527                                 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1528                         data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1529                                 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1530                         if (orig != data)
1531                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1532
1533                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1534                         data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1535                                 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1536                         data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1537                                 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1538                         if (orig != data)
1539                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1540
1541                         orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1542                         data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1543                         data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1544                         if (orig != data)
1545                                 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1546
1547                         if (!disable_clkreq) {
1548                                 struct pci_dev *root = adev->pdev->bus->self;
1549                                 u32 lnkcap;
1550
1551                                 clk_req_support = false;
1552                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1553                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1554                                         clk_req_support = true;
1555                         } else {
1556                                 clk_req_support = false;
1557                         }
1558
1559                         if (clk_req_support) {
1560                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1561                                 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1562                                         PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1563                                 if (orig != data)
1564                                         WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1565
1566                                 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1567                                 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1568                                         THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1569                                 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1570                                         (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1571                                 if (orig != data)
1572                                         WREG32_SMC(ixTHM_CLK_CNTL, data);
1573
1574                                 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1575                                 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1576                                         MISC_CLK_CTRL__ZCLK_SEL_MASK);
1577                                 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1578                                         (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1579                                 if (orig != data)
1580                                         WREG32_SMC(ixMISC_CLK_CTRL, data);
1581
1582                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1583                                 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1584                                 if (orig != data)
1585                                         WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1586
1587                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1588                                 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1589                                 if (orig != data)
1590                                         WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1591
1592                                 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1593                                 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1594                                 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1595                                 if (orig != data)
1596                                         WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1597                         }
1598                 }
1599         } else {
1600                 if (orig != data)
1601                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1602         }
1603
1604         orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1605         data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1606                 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1607                 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1608         if (orig != data)
1609                 WREG32_PCIE(ixPCIE_CNTL2, data);
1610
1611         if (!disable_l0s) {
1612                 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1613                 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1614                                 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1615                         data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1616                         if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1617                         (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1618                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1619                                 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1620                                 if (orig != data)
1621                                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1622                         }
1623                 }
1624         }
1625 }
1626
1627 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1628 {
1629         return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1630                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1631 }
1632
1633 static void cik_detect_hw_virtualization(struct amdgpu_device *adev)
1634 {
1635         if (is_virtual_machine()) /* passthrough mode */
1636                 adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
1637 }
1638
1639 static const struct amdgpu_asic_funcs cik_asic_funcs =
1640 {
1641         .read_disabled_bios = &cik_read_disabled_bios,
1642         .read_bios_from_rom = &cik_read_bios_from_rom,
1643         .read_register = &cik_read_register,
1644         .reset = &cik_asic_reset,
1645         .set_vga_state = &cik_vga_set_state,
1646         .get_xclk = &cik_get_xclk,
1647         .set_uvd_clocks = &cik_set_uvd_clocks,
1648         .set_vce_clocks = &cik_set_vce_clocks,
1649         .get_config_memsize = &cik_get_config_memsize,
1650 };
1651
1652 static int cik_common_early_init(void *handle)
1653 {
1654         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1655
1656         adev->smc_rreg = &cik_smc_rreg;
1657         adev->smc_wreg = &cik_smc_wreg;
1658         adev->pcie_rreg = &cik_pcie_rreg;
1659         adev->pcie_wreg = &cik_pcie_wreg;
1660         adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1661         adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1662         adev->didt_rreg = &cik_didt_rreg;
1663         adev->didt_wreg = &cik_didt_wreg;
1664
1665         adev->asic_funcs = &cik_asic_funcs;
1666
1667         adev->rev_id = cik_get_rev_id(adev);
1668         adev->external_rev_id = 0xFF;
1669         switch (adev->asic_type) {
1670         case CHIP_BONAIRE:
1671                 adev->cg_flags =
1672                         AMD_CG_SUPPORT_GFX_MGCG |
1673                         AMD_CG_SUPPORT_GFX_MGLS |
1674                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1675                         AMD_CG_SUPPORT_GFX_CGLS |
1676                         AMD_CG_SUPPORT_GFX_CGTS |
1677                         AMD_CG_SUPPORT_GFX_CGTS_LS |
1678                         AMD_CG_SUPPORT_GFX_CP_LS |
1679                         AMD_CG_SUPPORT_MC_LS |
1680                         AMD_CG_SUPPORT_MC_MGCG |
1681                         AMD_CG_SUPPORT_SDMA_MGCG |
1682                         AMD_CG_SUPPORT_SDMA_LS |
1683                         AMD_CG_SUPPORT_BIF_LS |
1684                         AMD_CG_SUPPORT_VCE_MGCG |
1685                         AMD_CG_SUPPORT_UVD_MGCG |
1686                         AMD_CG_SUPPORT_HDP_LS |
1687                         AMD_CG_SUPPORT_HDP_MGCG;
1688                 adev->pg_flags = 0;
1689                 adev->external_rev_id = adev->rev_id + 0x14;
1690                 break;
1691         case CHIP_HAWAII:
1692                 adev->cg_flags =
1693                         AMD_CG_SUPPORT_GFX_MGCG |
1694                         AMD_CG_SUPPORT_GFX_MGLS |
1695                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1696                         AMD_CG_SUPPORT_GFX_CGLS |
1697                         AMD_CG_SUPPORT_GFX_CGTS |
1698                         AMD_CG_SUPPORT_GFX_CP_LS |
1699                         AMD_CG_SUPPORT_MC_LS |
1700                         AMD_CG_SUPPORT_MC_MGCG |
1701                         AMD_CG_SUPPORT_SDMA_MGCG |
1702                         AMD_CG_SUPPORT_SDMA_LS |
1703                         AMD_CG_SUPPORT_BIF_LS |
1704                         AMD_CG_SUPPORT_VCE_MGCG |
1705                         AMD_CG_SUPPORT_UVD_MGCG |
1706                         AMD_CG_SUPPORT_HDP_LS |
1707                         AMD_CG_SUPPORT_HDP_MGCG;
1708                 adev->pg_flags = 0;
1709                 adev->external_rev_id = 0x28;
1710                 break;
1711         case CHIP_KAVERI:
1712                 adev->cg_flags =
1713                         AMD_CG_SUPPORT_GFX_MGCG |
1714                         AMD_CG_SUPPORT_GFX_MGLS |
1715                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1716                         AMD_CG_SUPPORT_GFX_CGLS |
1717                         AMD_CG_SUPPORT_GFX_CGTS |
1718                         AMD_CG_SUPPORT_GFX_CGTS_LS |
1719                         AMD_CG_SUPPORT_GFX_CP_LS |
1720                         AMD_CG_SUPPORT_SDMA_MGCG |
1721                         AMD_CG_SUPPORT_SDMA_LS |
1722                         AMD_CG_SUPPORT_BIF_LS |
1723                         AMD_CG_SUPPORT_VCE_MGCG |
1724                         AMD_CG_SUPPORT_UVD_MGCG |
1725                         AMD_CG_SUPPORT_HDP_LS |
1726                         AMD_CG_SUPPORT_HDP_MGCG;
1727                 adev->pg_flags =
1728                         /*AMD_PG_SUPPORT_GFX_PG |
1729                           AMD_PG_SUPPORT_GFX_SMG |
1730                           AMD_PG_SUPPORT_GFX_DMG |*/
1731                         AMD_PG_SUPPORT_UVD |
1732                         AMD_PG_SUPPORT_VCE |
1733                         /*  AMD_PG_SUPPORT_CP |
1734                           AMD_PG_SUPPORT_GDS |
1735                           AMD_PG_SUPPORT_RLC_SMU_HS |
1736                           AMD_PG_SUPPORT_ACP |
1737                           AMD_PG_SUPPORT_SAMU |*/
1738                         0;
1739                 if (adev->pdev->device == 0x1312 ||
1740                         adev->pdev->device == 0x1316 ||
1741                         adev->pdev->device == 0x1317)
1742                         adev->external_rev_id = 0x41;
1743                 else
1744                         adev->external_rev_id = 0x1;
1745                 break;
1746         case CHIP_KABINI:
1747         case CHIP_MULLINS:
1748                 adev->cg_flags =
1749                         AMD_CG_SUPPORT_GFX_MGCG |
1750                         AMD_CG_SUPPORT_GFX_MGLS |
1751                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1752                         AMD_CG_SUPPORT_GFX_CGLS |
1753                         AMD_CG_SUPPORT_GFX_CGTS |
1754                         AMD_CG_SUPPORT_GFX_CGTS_LS |
1755                         AMD_CG_SUPPORT_GFX_CP_LS |
1756                         AMD_CG_SUPPORT_SDMA_MGCG |
1757                         AMD_CG_SUPPORT_SDMA_LS |
1758                         AMD_CG_SUPPORT_BIF_LS |
1759                         AMD_CG_SUPPORT_VCE_MGCG |
1760                         AMD_CG_SUPPORT_UVD_MGCG |
1761                         AMD_CG_SUPPORT_HDP_LS |
1762                         AMD_CG_SUPPORT_HDP_MGCG;
1763                 adev->pg_flags =
1764                         /*AMD_PG_SUPPORT_GFX_PG |
1765                           AMD_PG_SUPPORT_GFX_SMG | */
1766                         AMD_PG_SUPPORT_UVD |
1767                         /*AMD_PG_SUPPORT_VCE |
1768                           AMD_PG_SUPPORT_CP |
1769                           AMD_PG_SUPPORT_GDS |
1770                           AMD_PG_SUPPORT_RLC_SMU_HS |
1771                           AMD_PG_SUPPORT_SAMU |*/
1772                         0;
1773                 if (adev->asic_type == CHIP_KABINI) {
1774                         if (adev->rev_id == 0)
1775                                 adev->external_rev_id = 0x81;
1776                         else if (adev->rev_id == 1)
1777                                 adev->external_rev_id = 0x82;
1778                         else if (adev->rev_id == 2)
1779                                 adev->external_rev_id = 0x85;
1780                 } else
1781                         adev->external_rev_id = adev->rev_id + 0xa1;
1782                 break;
1783         default:
1784                 /* FIXME: not supported yet */
1785                 return -EINVAL;
1786         }
1787
1788         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1789
1790         amdgpu_get_pcie_info(adev);
1791
1792         return 0;
1793 }
1794
1795 static int cik_common_sw_init(void *handle)
1796 {
1797         return 0;
1798 }
1799
1800 static int cik_common_sw_fini(void *handle)
1801 {
1802         return 0;
1803 }
1804
1805 static int cik_common_hw_init(void *handle)
1806 {
1807         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1808
1809         /* move the golden regs per IP block */
1810         cik_init_golden_registers(adev);
1811         /* enable pcie gen2/3 link */
1812         cik_pcie_gen3_enable(adev);
1813         /* enable aspm */
1814         cik_program_aspm(adev);
1815
1816         return 0;
1817 }
1818
1819 static int cik_common_hw_fini(void *handle)
1820 {
1821         return 0;
1822 }
1823
1824 static int cik_common_suspend(void *handle)
1825 {
1826         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1827
1828         return cik_common_hw_fini(adev);
1829 }
1830
1831 static int cik_common_resume(void *handle)
1832 {
1833         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1834
1835         return cik_common_hw_init(adev);
1836 }
1837
1838 static bool cik_common_is_idle(void *handle)
1839 {
1840         return true;
1841 }
1842
1843 static int cik_common_wait_for_idle(void *handle)
1844 {
1845         return 0;
1846 }
1847
1848 static int cik_common_soft_reset(void *handle)
1849 {
1850         /* XXX hard reset?? */
1851         return 0;
1852 }
1853
1854 static int cik_common_set_clockgating_state(void *handle,
1855                                             enum amd_clockgating_state state)
1856 {
1857         return 0;
1858 }
1859
1860 static int cik_common_set_powergating_state(void *handle,
1861                                             enum amd_powergating_state state)
1862 {
1863         return 0;
1864 }
1865
1866 static const struct amd_ip_funcs cik_common_ip_funcs = {
1867         .name = "cik_common",
1868         .early_init = cik_common_early_init,
1869         .late_init = NULL,
1870         .sw_init = cik_common_sw_init,
1871         .sw_fini = cik_common_sw_fini,
1872         .hw_init = cik_common_hw_init,
1873         .hw_fini = cik_common_hw_fini,
1874         .suspend = cik_common_suspend,
1875         .resume = cik_common_resume,
1876         .is_idle = cik_common_is_idle,
1877         .wait_for_idle = cik_common_wait_for_idle,
1878         .soft_reset = cik_common_soft_reset,
1879         .set_clockgating_state = cik_common_set_clockgating_state,
1880         .set_powergating_state = cik_common_set_powergating_state,
1881 };
1882
1883 static const struct amdgpu_ip_block_version cik_common_ip_block =
1884 {
1885         .type = AMD_IP_BLOCK_TYPE_COMMON,
1886         .major = 1,
1887         .minor = 0,
1888         .rev = 0,
1889         .funcs = &cik_common_ip_funcs,
1890 };
1891
1892 int cik_set_ip_blocks(struct amdgpu_device *adev)
1893 {
1894         cik_detect_hw_virtualization(adev);
1895
1896         switch (adev->asic_type) {
1897         case CHIP_BONAIRE:
1898                 amdgpu_ip_block_add(adev, &cik_common_ip_block);
1899                 amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
1900                 amdgpu_ip_block_add(adev, &cik_ih_ip_block);
1901                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
1902                 if (adev->enable_virtual_display)
1903                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
1904 #if defined(CONFIG_DRM_AMD_DC)
1905                 else if (amdgpu_device_has_dc_support(adev))
1906                         amdgpu_ip_block_add(adev, &dm_ip_block);
1907 #endif
1908                 else
1909                         amdgpu_ip_block_add(adev, &dce_v8_2_ip_block);
1910                 amdgpu_ip_block_add(adev, &gfx_v7_2_ip_block);
1911                 amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
1912                 amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
1913                 amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
1914                 break;
1915         case CHIP_HAWAII:
1916                 amdgpu_ip_block_add(adev, &cik_common_ip_block);
1917                 amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
1918                 amdgpu_ip_block_add(adev, &cik_ih_ip_block);
1919                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
1920                 if (adev->enable_virtual_display)
1921                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
1922 #if defined(CONFIG_DRM_AMD_DC)
1923                 else if (amdgpu_device_has_dc_support(adev))
1924                         amdgpu_ip_block_add(adev, &dm_ip_block);
1925 #endif
1926                 else
1927                         amdgpu_ip_block_add(adev, &dce_v8_5_ip_block);
1928                 amdgpu_ip_block_add(adev, &gfx_v7_3_ip_block);
1929                 amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
1930                 amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
1931                 amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
1932                 break;
1933         case CHIP_KAVERI:
1934                 amdgpu_ip_block_add(adev, &cik_common_ip_block);
1935                 amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
1936                 amdgpu_ip_block_add(adev, &cik_ih_ip_block);
1937                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
1938                 if (adev->enable_virtual_display)
1939                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
1940 #if defined(CONFIG_DRM_AMD_DC)
1941                 else if (amdgpu_device_has_dc_support(adev))
1942                         amdgpu_ip_block_add(adev, &dm_ip_block);
1943 #endif
1944                 else
1945                         amdgpu_ip_block_add(adev, &dce_v8_1_ip_block);
1946                 amdgpu_ip_block_add(adev, &gfx_v7_1_ip_block);
1947                 amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
1948                 amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
1949                 amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
1950                 break;
1951         case CHIP_KABINI:
1952         case CHIP_MULLINS:
1953                 amdgpu_ip_block_add(adev, &cik_common_ip_block);
1954                 amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
1955                 amdgpu_ip_block_add(adev, &cik_ih_ip_block);
1956                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
1957                 if (adev->enable_virtual_display)
1958                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
1959 #if defined(CONFIG_DRM_AMD_DC)
1960                 else if (amdgpu_device_has_dc_support(adev))
1961                         amdgpu_ip_block_add(adev, &dm_ip_block);
1962 #endif
1963                 else
1964                         amdgpu_ip_block_add(adev, &dce_v8_3_ip_block);
1965                 amdgpu_ip_block_add(adev, &gfx_v7_2_ip_block);
1966                 amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
1967                 amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
1968                 amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
1969                 break;
1970         default:
1971                 /* FIXME: not supported yet */
1972                 return -EINVAL;
1973         }
1974         return 0;
1975 }