drivers/gpu/drm/amd/amdgpu/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28
  29 #include <drm/amdgpu_drm.h>
  30
  31 #include "amdgpu.h"
  32 #include "amdgpu_atombios.h"
  33 #include "amdgpu_ih.h"
  34 #include "amdgpu_uvd.h"
  35 #include "amdgpu_vce.h"
  36 #include "cikd.h"
  37 #include "atom.h"
  38 #include "amd_pcie.h"
  39
  40 #include "cik.h"
  41 #include "gmc_v7_0.h"
  42 #include "cik_ih.h"
  43 #include "dce_v8_0.h"
  44 #include "gfx_v7_0.h"
  45 #include "cik_sdma.h"
  46 #include "uvd_v4_2.h"
  47 #include "vce_v2_0.h"
  48 #include "cik_dpm.h"
  49
  50 #include "uvd/uvd_4_2_d.h"
  51
  52 #include "smu/smu_7_0_1_d.h"
  53 #include "smu/smu_7_0_1_sh_mask.h"
  54
  55 #include "dce/dce_8_0_d.h"
  56 #include "dce/dce_8_0_sh_mask.h"
  57
  58 #include "bif/bif_4_1_d.h"
  59 #include "bif/bif_4_1_sh_mask.h"
  60
  61 #include "gca/gfx_7_2_d.h"
  62 #include "gca/gfx_7_2_enum.h"
  63 #include "gca/gfx_7_2_sh_mask.h"
  64
  65 #include "gmc/gmc_7_1_d.h"
  66 #include "gmc/gmc_7_1_sh_mask.h"
  67
  68 #include "oss/oss_2_0_d.h"
  69 #include "oss/oss_2_0_sh_mask.h"
  70
  71 #include "amdgpu_dm.h"
  72 #include "amdgpu_amdkfd.h"
  73 #include "dce_virtual.h"
  74
  75 static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
  76 {
  77         {
  78                 .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
  79                 .max_width = 2048,
  80                 .max_height = 1152,
  81                 .max_pixels_per_frame = 2048 * 1152,
  82                 .max_level = 0,
  83         },
  84 };
  85
  86 static const struct amdgpu_video_codecs cik_video_codecs_encode =
  87 {
  88         .codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
  89         .codec_array = cik_video_codecs_encode_array,
  90 };
  91
  92 static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
  93 {
  94         {
  95                 .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
  96                 .max_width = 2048,
  97                 .max_height = 1152,
  98                 .max_pixels_per_frame = 2048 * 1152,
  99                 .max_level = 3,
 100         },
 101         {
 102                 .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
 103                 .max_width = 2048,
 104                 .max_height = 1152,
 105                 .max_pixels_per_frame = 2048 * 1152,
 106                 .max_level = 5,
 107         },
 108         {
 109                 .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
 110                 .max_width = 2048,
 111                 .max_height = 1152,
 112                 .max_pixels_per_frame = 2048 * 1152,
 113                 .max_level = 41,
 114         },
 115         {
 116                 .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
 117                 .max_width = 2048,
 118                 .max_height = 1152,
 119                 .max_pixels_per_frame = 2048 * 1152,
 120                 .max_level = 4,
 121         },
 122 };
 123
 124 static const struct amdgpu_video_codecs cik_video_codecs_decode =
 125 {
 126         .codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
 127         .codec_array = cik_video_codecs_decode_array,
 128 };
 129
 130 static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
 131                                   const struct amdgpu_video_codecs **codecs)
 132 {
 133         switch (adev->asic_type) {
 134         case CHIP_BONAIRE:
 135         case CHIP_HAWAII:
 136         case CHIP_KAVERI:
 137         case CHIP_KABINI:
 138         case CHIP_MULLINS:
 139                 if (encode)
 140                         *codecs = &cik_video_codecs_encode;
 141                 else
 142                         *codecs = &cik_video_codecs_decode;
 143                 return 0;
 144         default:
 145                 return -EINVAL;
 146         }
 147 }
 148
 149 /*
 150  * Indirect registers accessor
 151  */
 152 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 153 {
 154         unsigned long flags;
 155         u32 r;
 156
 157         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 158         WREG32(mmPCIE_INDEX, reg);
 159         (void)RREG32(mmPCIE_INDEX);
 160         r = RREG32(mmPCIE_DATA);
 161         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 162         return r;
 163 }
 164
 165 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 166 {
 167         unsigned long flags;
 168
 169         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 170         WREG32(mmPCIE_INDEX, reg);
 171         (void)RREG32(mmPCIE_INDEX);
 172         WREG32(mmPCIE_DATA, v);
 173         (void)RREG32(mmPCIE_DATA);
 174         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 175 }
 176
 177 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 178 {
 179         unsigned long flags;
 180         u32 r;
 181
 182         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 183         WREG32(mmSMC_IND_INDEX_0, (reg));
 184         r = RREG32(mmSMC_IND_DATA_0);
 185         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 186         return r;
 187 }
 188
 189 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 190 {
 191         unsigned long flags;
 192
 193         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 194         WREG32(mmSMC_IND_INDEX_0, (reg));
 195         WREG32(mmSMC_IND_DATA_0, (v));
 196         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 197 }
 198
 199 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 200 {
 201         unsigned long flags;
 202         u32 r;
 203
 204         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 205         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 206         r = RREG32(mmUVD_CTX_DATA);
 207         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 208         return r;
 209 }
 210
 211 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 212 {
 213         unsigned long flags;
 214
 215         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 216         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 217         WREG32(mmUVD_CTX_DATA, (v));
 218         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 219 }
 220
 221 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 222 {
 223         unsigned long flags;
 224         u32 r;
 225
 226         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 227         WREG32(mmDIDT_IND_INDEX, (reg));
 228         r = RREG32(mmDIDT_IND_DATA);
 229         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 230         return r;
 231 }
 232
 233 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 234 {
 235         unsigned long flags;
 236
 237         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 238         WREG32(mmDIDT_IND_INDEX, (reg));
 239         WREG32(mmDIDT_IND_DATA, (v));
 240         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 241 }
 242
 243 static const u32 bonaire_golden_spm_registers[] =
 244 {
 245         0xc200, 0xe0ffffff, 0xe0000000
 246 };
 247
 248 static const u32 bonaire_golden_common_registers[] =
 249 {
 250         0x31dc, 0xffffffff, 0x00000800,
 251         0x31dd, 0xffffffff, 0x00000800,
 252         0x31e6, 0xffffffff, 0x00007fbf,
 253         0x31e7, 0xffffffff, 0x00007faf
 254 };
 255
 256 static const u32 bonaire_golden_registers[] =
 257 {
 258         0xcd5, 0x00000333, 0x00000333,
 259         0xcd4, 0x000c0fc0, 0x00040200,
 260         0x2684, 0x00010000, 0x00058208,
 261         0xf000, 0xffff1fff, 0x00140000,
 262         0xf080, 0xfdfc0fff, 0x00000100,
 263         0xf08d, 0x40000000, 0x40000200,
 264         0x260c, 0xffffffff, 0x00000000,
 265         0x260d, 0xf00fffff, 0x00000400,
 266         0x260e, 0x0002021c, 0x00020200,
 267         0x31e, 0x00000080, 0x00000000,
 268         0x16ec, 0x000000f0, 0x00000070,
 269         0x16f0, 0xf0311fff, 0x80300000,
 270         0x263e, 0x73773777, 0x12010001,
 271         0xd43, 0x00810000, 0x408af000,
 272         0x1c0c, 0x31000111, 0x00000011,
 273         0xbd2, 0x73773777, 0x12010001,
 274         0x883, 0x00007fb6, 0x0021a1b1,
 275         0x884, 0x00007fb6, 0x002021b1,
 276         0x860, 0x00007fb6, 0x00002191,
 277         0x886, 0x00007fb6, 0x002121b1,
 278         0x887, 0x00007fb6, 0x002021b1,
 279         0x877, 0x00007fb6, 0x00002191,
 280         0x878, 0x00007fb6, 0x00002191,
 281         0xd8a, 0x0000003f, 0x0000000a,
 282         0xd8b, 0x0000003f, 0x0000000a,
 283         0xab9, 0x00073ffe, 0x000022a2,
 284         0x903, 0x000007ff, 0x00000000,
 285         0x2285, 0xf000003f, 0x00000007,
 286         0x22fc, 0x00002001, 0x00000001,
 287         0x22c9, 0xffffffff, 0x00ffffff,
 288         0xc281, 0x0000ff0f, 0x00000000,
 289         0xa293, 0x07ffffff, 0x06000000,
 290         0x136, 0x00000fff, 0x00000100,
 291         0xf9e, 0x00000001, 0x00000002,
 292         0x2440, 0x03000000, 0x0362c688,
 293         0x2300, 0x000000ff, 0x00000001,
 294         0x390, 0x00001fff, 0x00001fff,
 295         0x2418, 0x0000007f, 0x00000020,
 296         0x2542, 0x00010000, 0x00010000,
 297         0x2b05, 0x000003ff, 0x000000f3,
 298         0x2b03, 0xffffffff, 0x00001032
 299 };
 300
 301 static const u32 bonaire_mgcg_cgcg_init[] =
 302 {
 303         0x3108, 0xffffffff, 0xfffffffc,
 304         0xc200, 0xffffffff, 0xe0000000,
 305         0xf0a8, 0xffffffff, 0x00000100,
 306         0xf082, 0xffffffff, 0x00000100,
 307         0xf0b0, 0xffffffff, 0xc0000100,
 308         0xf0b2, 0xffffffff, 0xc0000100,
 309         0xf0b1, 0xffffffff, 0xc0000100,
 310         0x1579, 0xffffffff, 0x00600100,
 311         0xf0a0, 0xffffffff, 0x00000100,
 312         0xf085, 0xffffffff, 0x06000100,
 313         0xf088, 0xffffffff, 0x00000100,
 314         0xf086, 0xffffffff, 0x06000100,
 315         0xf081, 0xffffffff, 0x00000100,
 316         0xf0b8, 0xffffffff, 0x00000100,
 317         0xf089, 0xffffffff, 0x00000100,
 318         0xf080, 0xffffffff, 0x00000100,
 319         0xf08c, 0xffffffff, 0x00000100,
 320         0xf08d, 0xffffffff, 0x00000100,
 321         0xf094, 0xffffffff, 0x00000100,
 322         0xf095, 0xffffffff, 0x00000100,
 323         0xf096, 0xffffffff, 0x00000100,
 324         0xf097, 0xffffffff, 0x00000100,
 325         0xf098, 0xffffffff, 0x00000100,
 326         0xf09f, 0xffffffff, 0x00000100,
 327         0xf09e, 0xffffffff, 0x00000100,
 328         0xf084, 0xffffffff, 0x06000100,
 329         0xf0a4, 0xffffffff, 0x00000100,
 330         0xf09d, 0xffffffff, 0x00000100,
 331         0xf0ad, 0xffffffff, 0x00000100,
 332         0xf0ac, 0xffffffff, 0x00000100,
 333         0xf09c, 0xffffffff, 0x00000100,
 334         0xc200, 0xffffffff, 0xe0000000,
 335         0xf008, 0xffffffff, 0x00010000,
 336         0xf009, 0xffffffff, 0x00030002,
 337         0xf00a, 0xffffffff, 0x00040007,
 338         0xf00b, 0xffffffff, 0x00060005,
 339         0xf00c, 0xffffffff, 0x00090008,
 340         0xf00d, 0xffffffff, 0x00010000,
 341         0xf00e, 0xffffffff, 0x00030002,
 342         0xf00f, 0xffffffff, 0x00040007,
 343         0xf010, 0xffffffff, 0x00060005,
 344         0xf011, 0xffffffff, 0x00090008,
 345         0xf012, 0xffffffff, 0x00010000,
 346         0xf013, 0xffffffff, 0x00030002,
 347         0xf014, 0xffffffff, 0x00040007,
 348         0xf015, 0xffffffff, 0x00060005,
 349         0xf016, 0xffffffff, 0x00090008,
 350         0xf017, 0xffffffff, 0x00010000,
 351         0xf018, 0xffffffff, 0x00030002,
 352         0xf019, 0xffffffff, 0x00040007,
 353         0xf01a, 0xffffffff, 0x00060005,
 354         0xf01b, 0xffffffff, 0x00090008,
 355         0xf01c, 0xffffffff, 0x00010000,
 356         0xf01d, 0xffffffff, 0x00030002,
 357         0xf01e, 0xffffffff, 0x00040007,
 358         0xf01f, 0xffffffff, 0x00060005,
 359         0xf020, 0xffffffff, 0x00090008,
 360         0xf021, 0xffffffff, 0x00010000,
 361         0xf022, 0xffffffff, 0x00030002,
 362         0xf023, 0xffffffff, 0x00040007,
 363         0xf024, 0xffffffff, 0x00060005,
 364         0xf025, 0xffffffff, 0x00090008,
 365         0xf026, 0xffffffff, 0x00010000,
 366         0xf027, 0xffffffff, 0x00030002,
 367         0xf028, 0xffffffff, 0x00040007,
 368         0xf029, 0xffffffff, 0x00060005,
 369         0xf02a, 0xffffffff, 0x00090008,
 370         0xf000, 0xffffffff, 0x96e00200,
 371         0x21c2, 0xffffffff, 0x00900100,
 372         0x3109, 0xffffffff, 0x0020003f,
 373         0xe, 0xffffffff, 0x0140001c,
 374         0xf, 0x000f0000, 0x000f0000,
 375         0x88, 0xffffffff, 0xc060000c,
 376         0x89, 0xc0000fff, 0x00000100,
 377         0x3e4, 0xffffffff, 0x00000100,
 378         0x3e6, 0x00000101, 0x00000000,
 379         0x82a, 0xffffffff, 0x00000104,
 380         0x1579, 0xff000fff, 0x00000100,
 381         0xc33, 0xc0000fff, 0x00000104,
 382         0x3079, 0x00000001, 0x00000001,
 383         0x3403, 0xff000ff0, 0x00000100,
 384         0x3603, 0xff000ff0, 0x00000100
 385 };
 386
 387 static const u32 spectre_golden_spm_registers[] =
 388 {
 389         0xc200, 0xe0ffffff, 0xe0000000
 390 };
 391
 392 static const u32 spectre_golden_common_registers[] =
 393 {
 394         0x31dc, 0xffffffff, 0x00000800,
 395         0x31dd, 0xffffffff, 0x00000800,
 396         0x31e6, 0xffffffff, 0x00007fbf,
 397         0x31e7, 0xffffffff, 0x00007faf
 398 };
 399
 400 static const u32 spectre_golden_registers[] =
 401 {
 402         0xf000, 0xffff1fff, 0x96940200,
 403         0xf003, 0xffff0001, 0xff000000,
 404         0xf080, 0xfffc0fff, 0x00000100,
 405         0x1bb6, 0x00010101, 0x00010000,
 406         0x260d, 0xf00fffff, 0x00000400,
 407         0x260e, 0xfffffffc, 0x00020200,
 408         0x16ec, 0x000000f0, 0x00000070,
 409         0x16f0, 0xf0311fff, 0x80300000,
 410         0x263e, 0x73773777, 0x12010001,
 411         0x26df, 0x00ff0000, 0x00fc0000,
 412         0xbd2, 0x73773777, 0x12010001,
 413         0x2285, 0xf000003f, 0x00000007,
 414         0x22c9, 0xffffffff, 0x00ffffff,
 415         0xa0d4, 0x3f3f3fff, 0x00000082,
 416         0xa0d5, 0x0000003f, 0x00000000,
 417         0xf9e, 0x00000001, 0x00000002,
 418         0x244f, 0xffff03df, 0x00000004,
 419         0x31da, 0x00000008, 0x00000008,
 420         0x2300, 0x000008ff, 0x00000800,
 421         0x2542, 0x00010000, 0x00010000,
 422         0x2b03, 0xffffffff, 0x54763210,
 423         0x853e, 0x01ff01ff, 0x00000002,
 424         0x8526, 0x007ff800, 0x00200000,
 425         0x8057, 0xffffffff, 0x00000f40,
 426         0xc24d, 0xffffffff, 0x00000001
 427 };
 428
 429 static const u32 spectre_mgcg_cgcg_init[] =
 430 {
 431         0x3108, 0xffffffff, 0xfffffffc,
 432         0xc200, 0xffffffff, 0xe0000000,
 433         0xf0a8, 0xffffffff, 0x00000100,
 434         0xf082, 0xffffffff, 0x00000100,
 435         0xf0b0, 0xffffffff, 0x00000100,
 436         0xf0b2, 0xffffffff, 0x00000100,
 437         0xf0b1, 0xffffffff, 0x00000100,
 438         0x1579, 0xffffffff, 0x00600100,
 439         0xf0a0, 0xffffffff, 0x00000100,
 440         0xf085, 0xffffffff, 0x06000100,
 441         0xf088, 0xffffffff, 0x00000100,
 442         0xf086, 0xffffffff, 0x06000100,
 443         0xf081, 0xffffffff, 0x00000100,
 444         0xf0b8, 0xffffffff, 0x00000100,
 445         0xf089, 0xffffffff, 0x00000100,
 446         0xf080, 0xffffffff, 0x00000100,
 447         0xf08c, 0xffffffff, 0x00000100,
 448         0xf08d, 0xffffffff, 0x00000100,
 449         0xf094, 0xffffffff, 0x00000100,
 450         0xf095, 0xffffffff, 0x00000100,
 451         0xf096, 0xffffffff, 0x00000100,
 452         0xf097, 0xffffffff, 0x00000100,
 453         0xf098, 0xffffffff, 0x00000100,
 454         0xf09f, 0xffffffff, 0x00000100,
 455         0xf09e, 0xffffffff, 0x00000100,
 456         0xf084, 0xffffffff, 0x06000100,
 457         0xf0a4, 0xffffffff, 0x00000100,
 458         0xf09d, 0xffffffff, 0x00000100,
 459         0xf0ad, 0xffffffff, 0x00000100,
 460         0xf0ac, 0xffffffff, 0x00000100,
 461         0xf09c, 0xffffffff, 0x00000100,
 462         0xc200, 0xffffffff, 0xe0000000,
 463         0xf008, 0xffffffff, 0x00010000,
 464         0xf009, 0xffffffff, 0x00030002,
 465         0xf00a, 0xffffffff, 0x00040007,
 466         0xf00b, 0xffffffff, 0x00060005,
 467         0xf00c, 0xffffffff, 0x00090008,
 468         0xf00d, 0xffffffff, 0x00010000,
 469         0xf00e, 0xffffffff, 0x00030002,
 470         0xf00f, 0xffffffff, 0x00040007,
 471         0xf010, 0xffffffff, 0x00060005,
 472         0xf011, 0xffffffff, 0x00090008,
 473         0xf012, 0xffffffff, 0x00010000,
 474         0xf013, 0xffffffff, 0x00030002,
 475         0xf014, 0xffffffff, 0x00040007,
 476         0xf015, 0xffffffff, 0x00060005,
 477         0xf016, 0xffffffff, 0x00090008,
 478         0xf017, 0xffffffff, 0x00010000,
 479         0xf018, 0xffffffff, 0x00030002,
 480         0xf019, 0xffffffff, 0x00040007,
 481         0xf01a, 0xffffffff, 0x00060005,
 482         0xf01b, 0xffffffff, 0x00090008,
 483         0xf01c, 0xffffffff, 0x00010000,
 484         0xf01d, 0xffffffff, 0x00030002,
 485         0xf01e, 0xffffffff, 0x00040007,
 486         0xf01f, 0xffffffff, 0x00060005,
 487         0xf020, 0xffffffff, 0x00090008,
 488         0xf021, 0xffffffff, 0x00010000,
 489         0xf022, 0xffffffff, 0x00030002,
 490         0xf023, 0xffffffff, 0x00040007,
 491         0xf024, 0xffffffff, 0x00060005,
 492         0xf025, 0xffffffff, 0x00090008,
 493         0xf026, 0xffffffff, 0x00010000,
 494         0xf027, 0xffffffff, 0x00030002,
 495         0xf028, 0xffffffff, 0x00040007,
 496         0xf029, 0xffffffff, 0x00060005,
 497         0xf02a, 0xffffffff, 0x00090008,
 498         0xf02b, 0xffffffff, 0x00010000,
 499         0xf02c, 0xffffffff, 0x00030002,
 500         0xf02d, 0xffffffff, 0x00040007,
 501         0xf02e, 0xffffffff, 0x00060005,
 502         0xf02f, 0xffffffff, 0x00090008,
 503         0xf000, 0xffffffff, 0x96e00200,
 504         0x21c2, 0xffffffff, 0x00900100,
 505         0x3109, 0xffffffff, 0x0020003f,
 506         0xe, 0xffffffff, 0x0140001c,
 507         0xf, 0x000f0000, 0x000f0000,
 508         0x88, 0xffffffff, 0xc060000c,
 509         0x89, 0xc0000fff, 0x00000100,
 510         0x3e4, 0xffffffff, 0x00000100,
 511         0x3e6, 0x00000101, 0x00000000,
 512         0x82a, 0xffffffff, 0x00000104,
 513         0x1579, 0xff000fff, 0x00000100,
 514         0xc33, 0xc0000fff, 0x00000104,
 515         0x3079, 0x00000001, 0x00000001,
 516         0x3403, 0xff000ff0, 0x00000100,
 517         0x3603, 0xff000ff0, 0x00000100
 518 };
 519
 520 static const u32 kalindi_golden_spm_registers[] =
 521 {
 522         0xc200, 0xe0ffffff, 0xe0000000
 523 };
 524
 525 static const u32 kalindi_golden_common_registers[] =
 526 {
 527         0x31dc, 0xffffffff, 0x00000800,
 528         0x31dd, 0xffffffff, 0x00000800,
 529         0x31e6, 0xffffffff, 0x00007fbf,
 530         0x31e7, 0xffffffff, 0x00007faf
 531 };
 532
 533 static const u32 kalindi_golden_registers[] =
 534 {
 535         0xf000, 0xffffdfff, 0x6e944040,
 536         0x1579, 0xff607fff, 0xfc000100,
 537         0xf088, 0xff000fff, 0x00000100,
 538         0xf089, 0xff000fff, 0x00000100,
 539         0xf080, 0xfffc0fff, 0x00000100,
 540         0x1bb6, 0x00010101, 0x00010000,
 541         0x260c, 0xffffffff, 0x00000000,
 542         0x260d, 0xf00fffff, 0x00000400,
 543         0x16ec, 0x000000f0, 0x00000070,
 544         0x16f0, 0xf0311fff, 0x80300000,
 545         0x263e, 0x73773777, 0x12010001,
 546         0x263f, 0xffffffff, 0x00000010,
 547         0x26df, 0x00ff0000, 0x00fc0000,
 548         0x200c, 0x00001f0f, 0x0000100a,
 549         0xbd2, 0x73773777, 0x12010001,
 550         0x902, 0x000fffff, 0x000c007f,
 551         0x2285, 0xf000003f, 0x00000007,
 552         0x22c9, 0x3fff3fff, 0x00ffcfff,
 553         0xc281, 0x0000ff0f, 0x00000000,
 554         0xa293, 0x07ffffff, 0x06000000,
 555         0x136, 0x00000fff, 0x00000100,
 556         0xf9e, 0x00000001, 0x00000002,
 557         0x31da, 0x00000008, 0x00000008,
 558         0x2300, 0x000000ff, 0x00000003,
 559         0x853e, 0x01ff01ff, 0x00000002,
 560         0x8526, 0x007ff800, 0x00200000,
 561         0x8057, 0xffffffff, 0x00000f40,
 562         0x2231, 0x001f3ae3, 0x00000082,
 563         0x2235, 0x0000001f, 0x00000010,
 564         0xc24d, 0xffffffff, 0x00000000
 565 };
 566
 567 static const u32 kalindi_mgcg_cgcg_init[] =
 568 {
 569         0x3108, 0xffffffff, 0xfffffffc,
 570         0xc200, 0xffffffff, 0xe0000000,
 571         0xf0a8, 0xffffffff, 0x00000100,
 572         0xf082, 0xffffffff, 0x00000100,
 573         0xf0b0, 0xffffffff, 0x00000100,
 574         0xf0b2, 0xffffffff, 0x00000100,
 575         0xf0b1, 0xffffffff, 0x00000100,
 576         0x1579, 0xffffffff, 0x00600100,
 577         0xf0a0, 0xffffffff, 0x00000100,
 578         0xf085, 0xffffffff, 0x06000100,
 579         0xf088, 0xffffffff, 0x00000100,
 580         0xf086, 0xffffffff, 0x06000100,
 581         0xf081, 0xffffffff, 0x00000100,
 582         0xf0b8, 0xffffffff, 0x00000100,
 583         0xf089, 0xffffffff, 0x00000100,
 584         0xf080, 0xffffffff, 0x00000100,
 585         0xf08c, 0xffffffff, 0x00000100,
 586         0xf08d, 0xffffffff, 0x00000100,
 587         0xf094, 0xffffffff, 0x00000100,
 588         0xf095, 0xffffffff, 0x00000100,
 589         0xf096, 0xffffffff, 0x00000100,
 590         0xf097, 0xffffffff, 0x00000100,
 591         0xf098, 0xffffffff, 0x00000100,
 592         0xf09f, 0xffffffff, 0x00000100,
 593         0xf09e, 0xffffffff, 0x00000100,
 594         0xf084, 0xffffffff, 0x06000100,
 595         0xf0a4, 0xffffffff, 0x00000100,
 596         0xf09d, 0xffffffff, 0x00000100,
 597         0xf0ad, 0xffffffff, 0x00000100,
 598         0xf0ac, 0xffffffff, 0x00000100,
 599         0xf09c, 0xffffffff, 0x00000100,
 600         0xc200, 0xffffffff, 0xe0000000,
 601         0xf008, 0xffffffff, 0x00010000,
 602         0xf009, 0xffffffff, 0x00030002,
 603         0xf00a, 0xffffffff, 0x00040007,
 604         0xf00b, 0xffffffff, 0x00060005,
 605         0xf00c, 0xffffffff, 0x00090008,
 606         0xf00d, 0xffffffff, 0x00010000,
 607         0xf00e, 0xffffffff, 0x00030002,
 608         0xf00f, 0xffffffff, 0x00040007,
 609         0xf010, 0xffffffff, 0x00060005,
 610         0xf011, 0xffffffff, 0x00090008,
 611         0xf000, 0xffffffff, 0x96e00200,
 612         0x21c2, 0xffffffff, 0x00900100,
 613         0x3109, 0xffffffff, 0x0020003f,
 614         0xe, 0xffffffff, 0x0140001c,
 615         0xf, 0x000f0000, 0x000f0000,
 616         0x88, 0xffffffff, 0xc060000c,
 617         0x89, 0xc0000fff, 0x00000100,
 618         0x82a, 0xffffffff, 0x00000104,
 619         0x1579, 0xff000fff, 0x00000100,
 620         0xc33, 0xc0000fff, 0x00000104,
 621         0x3079, 0x00000001, 0x00000001,
 622         0x3403, 0xff000ff0, 0x00000100,
 623         0x3603, 0xff000ff0, 0x00000100
 624 };
 625
 626 static const u32 hawaii_golden_spm_registers[] =
 627 {
 628         0xc200, 0xe0ffffff, 0xe0000000
 629 };
 630
 631 static const u32 hawaii_golden_common_registers[] =
 632 {
 633         0xc200, 0xffffffff, 0xe0000000,
 634         0xa0d4, 0xffffffff, 0x3a00161a,
 635         0xa0d5, 0xffffffff, 0x0000002e,
 636         0x2684, 0xffffffff, 0x00018208,
 637         0x263e, 0xffffffff, 0x12011003
 638 };
 639
 640 static const u32 hawaii_golden_registers[] =
 641 {
 642         0xcd5, 0x00000333, 0x00000333,
 643         0x2684, 0x00010000, 0x00058208,
 644         0x260c, 0xffffffff, 0x00000000,
 645         0x260d, 0xf00fffff, 0x00000400,
 646         0x260e, 0x0002021c, 0x00020200,
 647         0x31e, 0x00000080, 0x00000000,
 648         0x16ec, 0x000000f0, 0x00000070,
 649         0x16f0, 0xf0311fff, 0x80300000,
 650         0xd43, 0x00810000, 0x408af000,
 651         0x1c0c, 0x31000111, 0x00000011,
 652         0xbd2, 0x73773777, 0x12010001,
 653         0x848, 0x0000007f, 0x0000001b,
 654         0x877, 0x00007fb6, 0x00002191,
 655         0xd8a, 0x0000003f, 0x0000000a,
 656         0xd8b, 0x0000003f, 0x0000000a,
 657         0xab9, 0x00073ffe, 0x000022a2,
 658         0x903, 0x000007ff, 0x00000000,
 659         0x22fc, 0x00002001, 0x00000001,
 660         0x22c9, 0xffffffff, 0x00ffffff,
 661         0xc281, 0x0000ff0f, 0x00000000,
 662         0xa293, 0x07ffffff, 0x06000000,
 663         0xf9e, 0x00000001, 0x00000002,
 664         0x31da, 0x00000008, 0x00000008,
 665         0x31dc, 0x00000f00, 0x00000800,
 666         0x31dd, 0x00000f00, 0x00000800,
 667         0x31e6, 0x00ffffff, 0x00ff7fbf,
 668         0x31e7, 0x00ffffff, 0x00ff7faf,
 669         0x2300, 0x000000ff, 0x00000800,
 670         0x390, 0x00001fff, 0x00001fff,
 671         0x2418, 0x0000007f, 0x00000020,
 672         0x2542, 0x00010000, 0x00010000,
 673         0x2b80, 0x00100000, 0x000ff07c,
 674         0x2b05, 0x000003ff, 0x0000000f,
 675         0x2b04, 0xffffffff, 0x7564fdec,
 676         0x2b03, 0xffffffff, 0x3120b9a8,
 677         0x2b02, 0x20000000, 0x0f9c0000
 678 };
 679
 680 static const u32 hawaii_mgcg_cgcg_init[] =
 681 {
 682         0x3108, 0xffffffff, 0xfffffffd,
 683         0xc200, 0xffffffff, 0xe0000000,
 684         0xf0a8, 0xffffffff, 0x00000100,
 685         0xf082, 0xffffffff, 0x00000100,
 686         0xf0b0, 0xffffffff, 0x00000100,
 687         0xf0b2, 0xffffffff, 0x00000100,
 688         0xf0b1, 0xffffffff, 0x00000100,
 689         0x1579, 0xffffffff, 0x00200100,
 690         0xf0a0, 0xffffffff, 0x00000100,
 691         0xf085, 0xffffffff, 0x06000100,
 692         0xf088, 0xffffffff, 0x00000100,
 693         0xf086, 0xffffffff, 0x06000100,
 694         0xf081, 0xffffffff, 0x00000100,
 695         0xf0b8, 0xffffffff, 0x00000100,
 696         0xf089, 0xffffffff, 0x00000100,
 697         0xf080, 0xffffffff, 0x00000100,
 698         0xf08c, 0xffffffff, 0x00000100,
 699         0xf08d, 0xffffffff, 0x00000100,
 700         0xf094, 0xffffffff, 0x00000100,
 701         0xf095, 0xffffffff, 0x00000100,
 702         0xf096, 0xffffffff, 0x00000100,
 703         0xf097, 0xffffffff, 0x00000100,
 704         0xf098, 0xffffffff, 0x00000100,
 705         0xf09f, 0xffffffff, 0x00000100,
 706         0xf09e, 0xffffffff, 0x00000100,
 707         0xf084, 0xffffffff, 0x06000100,
 708         0xf0a4, 0xffffffff, 0x00000100,
 709         0xf09d, 0xffffffff, 0x00000100,
 710         0xf0ad, 0xffffffff, 0x00000100,
 711         0xf0ac, 0xffffffff, 0x00000100,
 712         0xf09c, 0xffffffff, 0x00000100,
 713         0xc200, 0xffffffff, 0xe0000000,
 714         0xf008, 0xffffffff, 0x00010000,
 715         0xf009, 0xffffffff, 0x00030002,
 716         0xf00a, 0xffffffff, 0x00040007,
 717         0xf00b, 0xffffffff, 0x00060005,
 718         0xf00c, 0xffffffff, 0x00090008,
 719         0xf00d, 0xffffffff, 0x00010000,
 720         0xf00e, 0xffffffff, 0x00030002,
 721         0xf00f, 0xffffffff, 0x00040007,
 722         0xf010, 0xffffffff, 0x00060005,
 723         0xf011, 0xffffffff, 0x00090008,
 724         0xf012, 0xffffffff, 0x00010000,
 725         0xf013, 0xffffffff, 0x00030002,
 726         0xf014, 0xffffffff, 0x00040007,
 727         0xf015, 0xffffffff, 0x00060005,
 728         0xf016, 0xffffffff, 0x00090008,
 729         0xf017, 0xffffffff, 0x00010000,
 730         0xf018, 0xffffffff, 0x00030002,
 731         0xf019, 0xffffffff, 0x00040007,
 732         0xf01a, 0xffffffff, 0x00060005,
 733         0xf01b, 0xffffffff, 0x00090008,
 734         0xf01c, 0xffffffff, 0x00010000,
 735         0xf01d, 0xffffffff, 0x00030002,
 736         0xf01e, 0xffffffff, 0x00040007,
 737         0xf01f, 0xffffffff, 0x00060005,
 738         0xf020, 0xffffffff, 0x00090008,
 739         0xf021, 0xffffffff, 0x00010000,
 740         0xf022, 0xffffffff, 0x00030002,
 741         0xf023, 0xffffffff, 0x00040007,
 742         0xf024, 0xffffffff, 0x00060005,
 743         0xf025, 0xffffffff, 0x00090008,
 744         0xf026, 0xffffffff, 0x00010000,
 745         0xf027, 0xffffffff, 0x00030002,
 746         0xf028, 0xffffffff, 0x00040007,
 747         0xf029, 0xffffffff, 0x00060005,
 748         0xf02a, 0xffffffff, 0x00090008,
 749         0xf02b, 0xffffffff, 0x00010000,
 750         0xf02c, 0xffffffff, 0x00030002,
 751         0xf02d, 0xffffffff, 0x00040007,
 752         0xf02e, 0xffffffff, 0x00060005,
 753         0xf02f, 0xffffffff, 0x00090008,
 754         0xf030, 0xffffffff, 0x00010000,
 755         0xf031, 0xffffffff, 0x00030002,
 756         0xf032, 0xffffffff, 0x00040007,
 757         0xf033, 0xffffffff, 0x00060005,
 758         0xf034, 0xffffffff, 0x00090008,
 759         0xf035, 0xffffffff, 0x00010000,
 760         0xf036, 0xffffffff, 0x00030002,
 761         0xf037, 0xffffffff, 0x00040007,
 762         0xf038, 0xffffffff, 0x00060005,
 763         0xf039, 0xffffffff, 0x00090008,
 764         0xf03a, 0xffffffff, 0x00010000,
 765         0xf03b, 0xffffffff, 0x00030002,
 766         0xf03c, 0xffffffff, 0x00040007,
 767         0xf03d, 0xffffffff, 0x00060005,
 768         0xf03e, 0xffffffff, 0x00090008,
 769         0x30c6, 0xffffffff, 0x00020200,
 770         0xcd4, 0xffffffff, 0x00000200,
 771         0x570, 0xffffffff, 0x00000400,
 772         0x157a, 0xffffffff, 0x00000000,
 773         0xbd4, 0xffffffff, 0x00000902,
 774         0xf000, 0xffffffff, 0x96940200,
 775         0x21c2, 0xffffffff, 0x00900100,
 776         0x3109, 0xffffffff, 0x0020003f,
 777         0xe, 0xffffffff, 0x0140001c,
 778         0xf, 0x000f0000, 0x000f0000,
 779         0x88, 0xffffffff, 0xc060000c,
 780         0x89, 0xc0000fff, 0x00000100,
 781         0x3e4, 0xffffffff, 0x00000100,
 782         0x3e6, 0x00000101, 0x00000000,
 783         0x82a, 0xffffffff, 0x00000104,
 784         0x1579, 0xff000fff, 0x00000100,
 785         0xc33, 0xc0000fff, 0x00000104,
 786         0x3079, 0x00000001, 0x00000001,
 787         0x3403, 0xff000ff0, 0x00000100,
 788         0x3603, 0xff000ff0, 0x00000100
 789 };
 790
 791 static const u32 godavari_golden_registers[] =
 792 {
 793         0x1579, 0xff607fff, 0xfc000100,
 794         0x1bb6, 0x00010101, 0x00010000,
 795         0x260c, 0xffffffff, 0x00000000,
 796         0x260c0, 0xf00fffff, 0x00000400,
 797         0x184c, 0xffffffff, 0x00010000,
 798         0x16ec, 0x000000f0, 0x00000070,
 799         0x16f0, 0xf0311fff, 0x80300000,
 800         0x263e, 0x73773777, 0x12010001,
 801         0x263f, 0xffffffff, 0x00000010,
 802         0x200c, 0x00001f0f, 0x0000100a,
 803         0xbd2, 0x73773777, 0x12010001,
 804         0x902, 0x000fffff, 0x000c007f,
 805         0x2285, 0xf000003f, 0x00000007,
 806         0x22c9, 0xffffffff, 0x00ff0fff,
 807         0xc281, 0x0000ff0f, 0x00000000,
 808         0xa293, 0x07ffffff, 0x06000000,
 809         0x136, 0x00000fff, 0x00000100,
 810         0x3405, 0x00010000, 0x00810001,
 811         0x3605, 0x00010000, 0x00810001,
 812         0xf9e, 0x00000001, 0x00000002,
 813         0x31da, 0x00000008, 0x00000008,
 814         0x31dc, 0x00000f00, 0x00000800,
 815         0x31dd, 0x00000f00, 0x00000800,
 816         0x31e6, 0x00ffffff, 0x00ff7fbf,
 817         0x31e7, 0x00ffffff, 0x00ff7faf,
 818         0x2300, 0x000000ff, 0x00000001,
 819         0x853e, 0x01ff01ff, 0x00000002,
 820         0x8526, 0x007ff800, 0x00200000,
 821         0x8057, 0xffffffff, 0x00000f40,
 822         0x2231, 0x001f3ae3, 0x00000082,
 823         0x2235, 0x0000001f, 0x00000010,
 824         0xc24d, 0xffffffff, 0x00000000
 825 };
 826
 827 static void cik_init_golden_registers(struct amdgpu_device *adev)
 828 {
 829         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
 830         mutex_lock(&adev->grbm_idx_mutex);
 831
 832         switch (adev->asic_type) {
 833         case CHIP_BONAIRE:
 834                 amdgpu_device_program_register_sequence(adev,
 835                                                         bonaire_mgcg_cgcg_init,
 836                                                         ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 837                 amdgpu_device_program_register_sequence(adev,
 838                                                         bonaire_golden_registers,
 839                                                         ARRAY_SIZE(bonaire_golden_registers));
 840                 amdgpu_device_program_register_sequence(adev,
 841                                                         bonaire_golden_common_registers,
 842                                                         ARRAY_SIZE(bonaire_golden_common_registers));
 843                 amdgpu_device_program_register_sequence(adev,
 844                                                         bonaire_golden_spm_registers,
 845                                                         ARRAY_SIZE(bonaire_golden_spm_registers));
 846                 break;
 847         case CHIP_KABINI:
 848                 amdgpu_device_program_register_sequence(adev,
 849                                                         kalindi_mgcg_cgcg_init,
 850                                                         ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 851                 amdgpu_device_program_register_sequence(adev,
 852                                                         kalindi_golden_registers,
 853                                                         ARRAY_SIZE(kalindi_golden_registers));
 854                 amdgpu_device_program_register_sequence(adev,
 855                                                         kalindi_golden_common_registers,
 856                                                         ARRAY_SIZE(kalindi_golden_common_registers));
 857                 amdgpu_device_program_register_sequence(adev,
 858                                                         kalindi_golden_spm_registers,
 859                                                         ARRAY_SIZE(kalindi_golden_spm_registers));
 860                 break;
 861         case CHIP_MULLINS:
 862                 amdgpu_device_program_register_sequence(adev,
 863                                                         kalindi_mgcg_cgcg_init,
 864                                                         ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 865                 amdgpu_device_program_register_sequence(adev,
 866                                                         godavari_golden_registers,
 867                                                         ARRAY_SIZE(godavari_golden_registers));
 868                 amdgpu_device_program_register_sequence(adev,
 869                                                         kalindi_golden_common_registers,
 870                                                         ARRAY_SIZE(kalindi_golden_common_registers));
 871                 amdgpu_device_program_register_sequence(adev,
 872                                                         kalindi_golden_spm_registers,
 873                                                         ARRAY_SIZE(kalindi_golden_spm_registers));
 874                 break;
 875         case CHIP_KAVERI:
 876                 amdgpu_device_program_register_sequence(adev,
 877                                                         spectre_mgcg_cgcg_init,
 878                                                         ARRAY_SIZE(spectre_mgcg_cgcg_init));
 879                 amdgpu_device_program_register_sequence(adev,
 880                                                         spectre_golden_registers,
 881                                                         ARRAY_SIZE(spectre_golden_registers));
 882                 amdgpu_device_program_register_sequence(adev,
 883                                                         spectre_golden_common_registers,
 884                                                         ARRAY_SIZE(spectre_golden_common_registers));
 885                 amdgpu_device_program_register_sequence(adev,
 886                                                         spectre_golden_spm_registers,
 887                                                         ARRAY_SIZE(spectre_golden_spm_registers));
 888                 break;
 889         case CHIP_HAWAII:
 890                 amdgpu_device_program_register_sequence(adev,
 891                                                         hawaii_mgcg_cgcg_init,
 892                                                         ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 893                 amdgpu_device_program_register_sequence(adev,
 894                                                         hawaii_golden_registers,
 895                                                         ARRAY_SIZE(hawaii_golden_registers));
 896                 amdgpu_device_program_register_sequence(adev,
 897                                                         hawaii_golden_common_registers,
 898                                                         ARRAY_SIZE(hawaii_golden_common_registers));
 899                 amdgpu_device_program_register_sequence(adev,
 900                                                         hawaii_golden_spm_registers,
 901                                                         ARRAY_SIZE(hawaii_golden_spm_registers));
 902                 break;
 903         default:
 904                 break;
 905         }
 906         mutex_unlock(&adev->grbm_idx_mutex);
 907 }
 908
 909 /**
 910  * cik_get_xclk - get the xclk
 911  *
 912  * @adev: amdgpu_device pointer
 913  *
 914  * Returns the reference clock used by the gfx engine
 915  * (CIK).
 916  */
 917 static u32 cik_get_xclk(struct amdgpu_device *adev)
 918 {
 919         u32 reference_clock = adev->clock.spll.reference_freq;
 920
 921         if (adev->flags & AMD_IS_APU) {
 922                 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 923                         return reference_clock / 2;
 924         } else {
 925                 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 926                         return reference_clock / 4;
 927         }
 928         return reference_clock;
 929 }
 930
 931 /**
 932  * cik_srbm_select - select specific register instances
 933  *
 934  * @adev: amdgpu_device pointer
 935  * @me: selected ME (micro engine)
 936  * @pipe: pipe
 937  * @queue: queue
 938  * @vmid: VMID
 939  *
 940  * Switches the currently active registers instances.  Some
 941  * registers are instanced per VMID, others are instanced per
 942  * me/pipe/queue combination.
 943  */
 944 void cik_srbm_select(struct amdgpu_device *adev,
 945                      u32 me, u32 pipe, u32 queue, u32 vmid)
 946 {
 947         u32 srbm_gfx_cntl =
 948                 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 949                 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 950                 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 951                 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 952         WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 953 }
 954
 955 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 956 {
 957         uint32_t tmp;
 958
 959         tmp = RREG32(mmCONFIG_CNTL);
 960         if (!state)
 961                 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 962         else
 963                 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 964         WREG32(mmCONFIG_CNTL, tmp);
 965 }
 966
 967 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 968 {
 969         u32 bus_cntl;
 970         u32 d1vga_control = 0;
 971         u32 d2vga_control = 0;
 972         u32 vga_render_control = 0;
 973         u32 rom_cntl;
 974         bool r;
 975
 976         bus_cntl = RREG32(mmBUS_CNTL);
 977         if (adev->mode_info.num_crtc) {
 978                 d1vga_control = RREG32(mmD1VGA_CONTROL);
 979                 d2vga_control = RREG32(mmD2VGA_CONTROL);
 980                 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 981         }
 982         rom_cntl = RREG32_SMC(ixROM_CNTL);
 983
 984         /* enable the rom */
 985         WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 986         if (adev->mode_info.num_crtc) {
 987                 /* Disable VGA mode */
 988                 WREG32(mmD1VGA_CONTROL,
 989                        (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 990                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 991                 WREG32(mmD2VGA_CONTROL,
 992                        (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 993                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 994                 WREG32(mmVGA_RENDER_CONTROL,
 995                        (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 996         }
 997         WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 998
 999         r = amdgpu_read_bios(adev);
1000
1001         /* restore regs */
1002         WREG32(mmBUS_CNTL, bus_cntl);
1003         if (adev->mode_info.num_crtc) {
1004                 WREG32(mmD1VGA_CONTROL, d1vga_control);
1005                 WREG32(mmD2VGA_CONTROL, d2vga_control);
1006                 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007         }
1008         WREG32_SMC(ixROM_CNTL, rom_cntl);
1009         return r;
1010 }
1011
1012 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013                                    u8 *bios, u32 length_bytes)
1014 {
1015         u32 *dw_ptr;
1016         unsigned long flags;
1017         u32 i, length_dw;
1018
1019         if (bios == NULL)
1020                 return false;
1021         if (length_bytes == 0)
1022                 return false;
1023         /* APU vbios image is part of sbios image */
1024         if (adev->flags & AMD_IS_APU)
1025                 return false;
1026
1027         dw_ptr = (u32 *)bios;
1028         length_dw = ALIGN(length_bytes, 4) / 4;
1029         /* take the smc lock since we are using the smc index */
1030         spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031         /* set rom index to 0 */
1032         WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033         WREG32(mmSMC_IND_DATA_0, 0);
1034         /* set index to data for continous read */
1035         WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036         for (i = 0; i < length_dw; i++)
1037                 dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039
1040         return true;
1041 }
1042
1043 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044         {mmGRBM_STATUS},
1045         {mmGRBM_STATUS2},
1046         {mmGRBM_STATUS_SE0},
1047         {mmGRBM_STATUS_SE1},
1048         {mmGRBM_STATUS_SE2},
1049         {mmGRBM_STATUS_SE3},
1050         {mmSRBM_STATUS},
1051         {mmSRBM_STATUS2},
1052         {mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053         {mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054         {mmCP_STAT},
1055         {mmCP_STALLED_STAT1},
1056         {mmCP_STALLED_STAT2},
1057         {mmCP_STALLED_STAT3},
1058         {mmCP_CPF_BUSY_STAT},
1059         {mmCP_CPF_STALLED_STAT1},
1060         {mmCP_CPF_STATUS},
1061         {mmCP_CPC_BUSY_STAT},
1062         {mmCP_CPC_STALLED_STAT1},
1063         {mmCP_CPC_STATUS},
1064         {mmGB_ADDR_CONFIG},
1065         {mmMC_ARB_RAMCFG},
1066         {mmGB_TILE_MODE0},
1067         {mmGB_TILE_MODE1},
1068         {mmGB_TILE_MODE2},
1069         {mmGB_TILE_MODE3},
1070         {mmGB_TILE_MODE4},
1071         {mmGB_TILE_MODE5},
1072         {mmGB_TILE_MODE6},
1073         {mmGB_TILE_MODE7},
1074         {mmGB_TILE_MODE8},
1075         {mmGB_TILE_MODE9},
1076         {mmGB_TILE_MODE10},
1077         {mmGB_TILE_MODE11},
1078         {mmGB_TILE_MODE12},
1079         {mmGB_TILE_MODE13},
1080         {mmGB_TILE_MODE14},
1081         {mmGB_TILE_MODE15},
1082         {mmGB_TILE_MODE16},
1083         {mmGB_TILE_MODE17},
1084         {mmGB_TILE_MODE18},
1085         {mmGB_TILE_MODE19},
1086         {mmGB_TILE_MODE20},
1087         {mmGB_TILE_MODE21},
1088         {mmGB_TILE_MODE22},
1089         {mmGB_TILE_MODE23},
1090         {mmGB_TILE_MODE24},
1091         {mmGB_TILE_MODE25},
1092         {mmGB_TILE_MODE26},
1093         {mmGB_TILE_MODE27},
1094         {mmGB_TILE_MODE28},
1095         {mmGB_TILE_MODE29},
1096         {mmGB_TILE_MODE30},
1097         {mmGB_TILE_MODE31},
1098         {mmGB_MACROTILE_MODE0},
1099         {mmGB_MACROTILE_MODE1},
1100         {mmGB_MACROTILE_MODE2},
1101         {mmGB_MACROTILE_MODE3},
1102         {mmGB_MACROTILE_MODE4},
1103         {mmGB_MACROTILE_MODE5},
1104         {mmGB_MACROTILE_MODE6},
1105         {mmGB_MACROTILE_MODE7},
1106         {mmGB_MACROTILE_MODE8},
1107         {mmGB_MACROTILE_MODE9},
1108         {mmGB_MACROTILE_MODE10},
1109         {mmGB_MACROTILE_MODE11},
1110         {mmGB_MACROTILE_MODE12},
1111         {mmGB_MACROTILE_MODE13},
1112         {mmGB_MACROTILE_MODE14},
1113         {mmGB_MACROTILE_MODE15},
1114         {mmCC_RB_BACKEND_DISABLE, true},
1115         {mmGC_USER_RB_BACKEND_DISABLE, true},
1116         {mmGB_BACKEND_MAP, false},
1117         {mmPA_SC_RASTER_CONFIG, true},
1118         {mmPA_SC_RASTER_CONFIG_1, true},
1119 };
1120
1121
1122 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123                                        bool indexed, u32 se_num,
1124                                        u32 sh_num, u32 reg_offset)
1125 {
1126         if (indexed) {
1127                 uint32_t val;
1128                 unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129                 unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130
1131                 switch (reg_offset) {
1132                 case mmCC_RB_BACKEND_DISABLE:
1133                         return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134                 case mmGC_USER_RB_BACKEND_DISABLE:
1135                         return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136                 case mmPA_SC_RASTER_CONFIG:
1137                         return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138                 case mmPA_SC_RASTER_CONFIG_1:
1139                         return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140                 }
1141
1142                 mutex_lock(&adev->grbm_idx_mutex);
1143                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144                         amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1145
1146                 val = RREG32(reg_offset);
1147
1148                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149                         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1150                 mutex_unlock(&adev->grbm_idx_mutex);
1151                 return val;
1152         } else {
1153                 unsigned idx;
1154
1155                 switch (reg_offset) {
1156                 case mmGB_ADDR_CONFIG:
1157                         return adev->gfx.config.gb_addr_config;
1158                 case mmMC_ARB_RAMCFG:
1159                         return adev->gfx.config.mc_arb_ramcfg;
1160                 case mmGB_TILE_MODE0:
1161                 case mmGB_TILE_MODE1:
1162                 case mmGB_TILE_MODE2:
1163                 case mmGB_TILE_MODE3:
1164                 case mmGB_TILE_MODE4:
1165                 case mmGB_TILE_MODE5:
1166                 case mmGB_TILE_MODE6:
1167                 case mmGB_TILE_MODE7:
1168                 case mmGB_TILE_MODE8:
1169                 case mmGB_TILE_MODE9:
1170                 case mmGB_TILE_MODE10:
1171                 case mmGB_TILE_MODE11:
1172                 case mmGB_TILE_MODE12:
1173                 case mmGB_TILE_MODE13:
1174                 case mmGB_TILE_MODE14:
1175                 case mmGB_TILE_MODE15:
1176                 case mmGB_TILE_MODE16:
1177                 case mmGB_TILE_MODE17:
1178                 case mmGB_TILE_MODE18:
1179                 case mmGB_TILE_MODE19:
1180                 case mmGB_TILE_MODE20:
1181                 case mmGB_TILE_MODE21:
1182                 case mmGB_TILE_MODE22:
1183                 case mmGB_TILE_MODE23:
1184                 case mmGB_TILE_MODE24:
1185                 case mmGB_TILE_MODE25:
1186                 case mmGB_TILE_MODE26:
1187                 case mmGB_TILE_MODE27:
1188                 case mmGB_TILE_MODE28:
1189                 case mmGB_TILE_MODE29:
1190                 case mmGB_TILE_MODE30:
1191                 case mmGB_TILE_MODE31:
1192                         idx = (reg_offset - mmGB_TILE_MODE0);
1193                         return adev->gfx.config.tile_mode_array[idx];
1194                 case mmGB_MACROTILE_MODE0:
1195                 case mmGB_MACROTILE_MODE1:
1196                 case mmGB_MACROTILE_MODE2:
1197                 case mmGB_MACROTILE_MODE3:
1198                 case mmGB_MACROTILE_MODE4:
1199                 case mmGB_MACROTILE_MODE5:
1200                 case mmGB_MACROTILE_MODE6:
1201                 case mmGB_MACROTILE_MODE7:
1202                 case mmGB_MACROTILE_MODE8:
1203                 case mmGB_MACROTILE_MODE9:
1204                 case mmGB_MACROTILE_MODE10:
1205                 case mmGB_MACROTILE_MODE11:
1206                 case mmGB_MACROTILE_MODE12:
1207                 case mmGB_MACROTILE_MODE13:
1208                 case mmGB_MACROTILE_MODE14:
1209                 case mmGB_MACROTILE_MODE15:
1210                         idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211                         return adev->gfx.config.macrotile_mode_array[idx];
1212                 default:
1213                         return RREG32(reg_offset);
1214                 }
1215         }
1216 }
1217
1218 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219                              u32 sh_num, u32 reg_offset, u32 *value)
1220 {
1221         uint32_t i;
1222
1223         *value = 0;
1224         for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225                 bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226
1227                 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228                         continue;
1229
1230                 *value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231                                                 reg_offset);
1232                 return 0;
1233         }
1234         return -EINVAL;
1235 }
1236
1237 struct kv_reset_save_regs {
1238         u32 gmcon_reng_execute;
1239         u32 gmcon_misc;
1240         u32 gmcon_misc3;
1241 };
1242
1243 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244                                    struct kv_reset_save_regs *save)
1245 {
1246         save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247         save->gmcon_misc = RREG32(mmGMCON_MISC);
1248         save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249
1250         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251                 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252         WREG32(mmGMCON_MISC, save->gmcon_misc &
1253                 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254                         GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255 }
1256
1257 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258                                       struct kv_reset_save_regs *save)
1259 {
1260         int i;
1261
1262         WREG32(mmGMCON_PGFSM_WRITE, 0);
1263         WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264
1265         for (i = 0; i < 5; i++)
1266                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1267
1268         WREG32(mmGMCON_PGFSM_WRITE, 0);
1269         WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270
1271         for (i = 0; i < 5; i++)
1272                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1273
1274         WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275         WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276
1277         for (i = 0; i < 5; i++)
1278                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1279
1280         WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281         WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282
1283         for (i = 0; i < 5; i++)
1284                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1285
1286         WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287         WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288
1289         for (i = 0; i < 5; i++)
1290                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1291
1292         WREG32(mmGMCON_PGFSM_WRITE, 0);
1293         WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294
1295         for (i = 0; i < 5; i++)
1296                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1297
1298         WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299         WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300
1301         for (i = 0; i < 5; i++)
1302                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1303
1304         WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305         WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306
1307         for (i = 0; i < 5; i++)
1308                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1309
1310         WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311         WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312
1313         for (i = 0; i < 5; i++)
1314                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1315
1316         WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317         WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318
1319         for (i = 0; i < 5; i++)
1320                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1321
1322         WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323         WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324
1325         WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326         WREG32(mmGMCON_MISC, save->gmcon_misc);
1327         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328 }
1329
1330 /**
1331  * cik_asic_pci_config_reset - soft reset GPU
1332  *
1333  * @adev: amdgpu_device pointer
1334  *
1335  * Use PCI Config method to reset the GPU.
1336  *
1337  * Returns 0 for success.
1338  */
1339 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340 {
1341         struct kv_reset_save_regs kv_save = { 0 };
1342         u32 i;
1343         int r = -EINVAL;
1344
1345         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346
1347         if (adev->flags & AMD_IS_APU)
1348                 kv_save_regs_for_reset(adev, &kv_save);
1349
1350         /* disable BM */
1351         pci_clear_master(adev->pdev);
1352         /* reset */
1353         amdgpu_device_pci_config_reset(adev);
1354
1355         udelay(100);
1356
1357         /* wait for asic to come out of reset */
1358         for (i = 0; i < adev->usec_timeout; i++) {
1359                 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360                         /* enable BM */
1361                         pci_set_master(adev->pdev);
1362                         adev->has_hw_reset = true;
1363                         r = 0;
1364                         break;
1365                 }
1366                 udelay(1);
1367         }
1368
1369         /* does asic init need to be run first??? */
1370         if (adev->flags & AMD_IS_APU)
1371                 kv_restore_regs_for_reset(adev, &kv_save);
1372
1373         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374
1375         return r;
1376 }
1377
1378 static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1379 {
1380         switch (adev->asic_type) {
1381         case CHIP_BONAIRE:
1382         case CHIP_HAWAII:
1383                 return amdgpu_dpm_is_baco_supported(adev);
1384         default:
1385                 return false;
1386         }
1387 }
1388
1389 static enum amd_reset_method
1390 cik_asic_reset_method(struct amdgpu_device *adev)
1391 {
1392         bool baco_reset;
1393
1394         if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395             amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396                 return amdgpu_reset_method;
1397
1398         if (amdgpu_reset_method != -1)
1399                 dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400                                   amdgpu_reset_method);
1401
1402         switch (adev->asic_type) {
1403         case CHIP_BONAIRE:
1404         case CHIP_HAWAII:
1405                 baco_reset = cik_asic_supports_baco(adev);
1406                 break;
1407         default:
1408                 baco_reset = false;
1409                 break;
1410         }
1411
1412         if (baco_reset)
1413                 return AMD_RESET_METHOD_BACO;
1414         else
1415                 return AMD_RESET_METHOD_LEGACY;
1416 }
1417
1418 /**
1419  * cik_asic_reset - soft reset GPU
1420  *
1421  * @adev: amdgpu_device pointer
1422  *
1423  * Look up which blocks are hung and attempt
1424  * to reset them.
1425  * Returns 0 for success.
1426  */
1427 static int cik_asic_reset(struct amdgpu_device *adev)
1428 {
1429         int r;
1430
1431         if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1432                 dev_info(adev->dev, "BACO reset\n");
1433                 r = amdgpu_dpm_baco_reset(adev);
1434         } else {
1435                 dev_info(adev->dev, "PCI CONFIG reset\n");
1436                 r = cik_asic_pci_config_reset(adev);
1437         }
1438
1439         return r;
1440 }
1441
1442 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1443 {
1444         return RREG32(mmCONFIG_MEMSIZE);
1445 }
1446
1447 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1448                               u32 cntl_reg, u32 status_reg)
1449 {
1450         int r, i;
1451         struct atom_clock_dividers dividers;
1452         uint32_t tmp;
1453
1454         r = amdgpu_atombios_get_clock_dividers(adev,
1455                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1456                                                clock, false, &dividers);
1457         if (r)
1458                 return r;
1459
1460         tmp = RREG32_SMC(cntl_reg);
1461         tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1462                 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1463         tmp |= dividers.post_divider;
1464         WREG32_SMC(cntl_reg, tmp);
1465
1466         for (i = 0; i < 100; i++) {
1467                 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1468                         break;
1469                 mdelay(10);
1470         }
1471         if (i == 100)
1472                 return -ETIMEDOUT;
1473
1474         return 0;
1475 }
1476
1477 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1478 {
1479         int r = 0;
1480
1481         r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1482         if (r)
1483                 return r;
1484
1485         r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1486         return r;
1487 }
1488
1489 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1490 {
1491         int r, i;
1492         struct atom_clock_dividers dividers;
1493         u32 tmp;
1494
1495         r = amdgpu_atombios_get_clock_dividers(adev,
1496                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1497                                                ecclk, false, &dividers);
1498         if (r)
1499                 return r;
1500
1501         for (i = 0; i < 100; i++) {
1502                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1503                         break;
1504                 mdelay(10);
1505         }
1506         if (i == 100)
1507                 return -ETIMEDOUT;
1508
1509         tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1510         tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1511                 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1512         tmp |= dividers.post_divider;
1513         WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1514
1515         for (i = 0; i < 100; i++) {
1516                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1517                         break;
1518                 mdelay(10);
1519         }
1520         if (i == 100)
1521                 return -ETIMEDOUT;
1522
1523         return 0;
1524 }
1525
1526 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1527 {
1528         struct pci_dev *root = adev->pdev->bus->self;
1529         u32 speed_cntl, current_data_rate;
1530         int i;
1531         u16 tmp16;
1532
1533         if (pci_is_root_bus(adev->pdev->bus))
1534                 return;
1535
1536         if (amdgpu_pcie_gen2 == 0)
1537                 return;
1538
1539         if (adev->flags & AMD_IS_APU)
1540                 return;
1541
1542         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1543                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1544                 return;
1545
1546         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1547         current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1548                 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1549         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1550                 if (current_data_rate == 2) {
1551                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1552                         return;
1553                 }
1554                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1555         } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1556                 if (current_data_rate == 1) {
1557                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1558                         return;
1559                 }
1560                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1561         }
1562
1563         if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1564                 return;
1565
1566         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1567                 /* re-try equalization if gen3 is not already enabled */
1568                 if (current_data_rate != 2) {
1569                         u16 bridge_cfg, gpu_cfg;
1570                         u16 bridge_cfg2, gpu_cfg2;
1571                         u32 max_lw, current_lw, tmp;
1572
1573                         pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1574                                                   &bridge_cfg);
1575                         pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
1576                                                   &gpu_cfg);
1577
1578                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1579                         pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
1580
1581                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1582                         pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
1583                                                    tmp16);
1584
1585                         tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1586                         max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1587                                 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1588                         current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1589                                 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1590
1591                         if (current_lw < max_lw) {
1592                                 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1593                                 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1594                                         tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1595                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1596                                         tmp |= (max_lw <<
1597                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1598                                         tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1599                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1600                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1601                                         WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1602                                 }
1603                         }
1604
1605                         for (i = 0; i < 10; i++) {
1606                                 /* check status */
1607                                 pcie_capability_read_word(adev->pdev,
1608                                                           PCI_EXP_DEVSTA,
1609                                                           &tmp16);
1610                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1611                                         break;
1612
1613                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1614                                                           &bridge_cfg);
1615                                 pcie_capability_read_word(adev->pdev,
1616                                                           PCI_EXP_LNKCTL,
1617                                                           &gpu_cfg);
1618
1619                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1620                                                           &bridge_cfg2);
1621                                 pcie_capability_read_word(adev->pdev,
1622                                                           PCI_EXP_LNKCTL2,
1623                                                           &gpu_cfg2);
1624
1625                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1626                                 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1627                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1628
1629                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1630                                 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1631                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1632
1633                                 msleep(100);
1634
1635                                 /* linkctl */
1636                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1637                                                           &tmp16);
1638                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1639                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1640                                 pcie_capability_write_word(root, PCI_EXP_LNKCTL,
1641                                                            tmp16);
1642
1643                                 pcie_capability_read_word(adev->pdev,
1644                                                           PCI_EXP_LNKCTL,
1645                                                           &tmp16);
1646                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1647                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1648                                 pcie_capability_write_word(adev->pdev,
1649                                                            PCI_EXP_LNKCTL,
1650                                                            tmp16);
1651
1652                                 /* linkctl2 */
1653                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1654                                                           &tmp16);
1655                                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1656                                            PCI_EXP_LNKCTL2_TX_MARGIN);
1657                                 tmp16 |= (bridge_cfg2 &
1658                                           (PCI_EXP_LNKCTL2_ENTER_COMP |
1659                                            PCI_EXP_LNKCTL2_TX_MARGIN));
1660                                 pcie_capability_write_word(root,
1661                                                            PCI_EXP_LNKCTL2,
1662                                                            tmp16);
1663
1664                                 pcie_capability_read_word(adev->pdev,
1665                                                           PCI_EXP_LNKCTL2,
1666                                                           &tmp16);
1667                                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1668                                            PCI_EXP_LNKCTL2_TX_MARGIN);
1669                                 tmp16 |= (gpu_cfg2 &
1670                                           (PCI_EXP_LNKCTL2_ENTER_COMP |
1671                                            PCI_EXP_LNKCTL2_TX_MARGIN));
1672                                 pcie_capability_write_word(adev->pdev,
1673                                                            PCI_EXP_LNKCTL2,
1674                                                            tmp16);
1675
1676                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1677                                 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1678                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1679                         }
1680                 }
1681         }
1682
1683         /* set the link speed */
1684         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1685                 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1686         speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1687         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1688
1689         pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1690         tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1691
1692         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1693                 tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1694         else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1695                 tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1696         else
1697                 tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1698         pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1699
1700         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1701         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1702         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1703
1704         for (i = 0; i < adev->usec_timeout; i++) {
1705                 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1706                 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1707                         break;
1708                 udelay(1);
1709         }
1710 }
1711
1712 static void cik_program_aspm(struct amdgpu_device *adev)
1713 {
1714         u32 data, orig;
1715         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1716         bool disable_clkreq = false;
1717
1718         if (amdgpu_aspm == 0)
1719                 return;
1720
1721         if (pci_is_root_bus(adev->pdev->bus))
1722                 return;
1723
1724         /* XXX double check APUs */
1725         if (adev->flags & AMD_IS_APU)
1726                 return;
1727
1728         orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1729         data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1730         data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1731                 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1732         if (orig != data)
1733                 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1734
1735         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1736         data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1737         if (orig != data)
1738                 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1739
1740         orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1741         data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1742         if (orig != data)
1743                 WREG32_PCIE(ixPCIE_P_CNTL, data);
1744
1745         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1746         data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1747                 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1748         data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1749         if (!disable_l0s)
1750                 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1751
1752         if (!disable_l1) {
1753                 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1754                 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1755                 if (orig != data)
1756                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1757
1758                 if (!disable_plloff_in_l1) {
1759                         bool clk_req_support;
1760
1761                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1762                         data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1763                                 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1764                         data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1765                                 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1766                         if (orig != data)
1767                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1768
1769                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1770                         data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1771                                 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1772                         data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1773                                 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1774                         if (orig != data)
1775                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1776
1777                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1778                         data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1779                                 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1780                         data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1781                                 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1782                         if (orig != data)
1783                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1784
1785                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1786                         data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1787                                 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1788                         data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1789                                 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1790                         if (orig != data)
1791                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1792
1793                         orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1794                         data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1795                         data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1796                         if (orig != data)
1797                                 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1798
1799                         if (!disable_clkreq) {
1800                                 struct pci_dev *root = adev->pdev->bus->self;
1801                                 u32 lnkcap;
1802
1803                                 clk_req_support = false;
1804                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1805                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1806                                         clk_req_support = true;
1807                         } else {
1808                                 clk_req_support = false;
1809                         }
1810
1811                         if (clk_req_support) {
1812                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1813                                 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1814                                         PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1815                                 if (orig != data)
1816                                         WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1817
1818                                 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1819                                 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1820                                         THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1821                                 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1822                                         (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1823                                 if (orig != data)
1824                                         WREG32_SMC(ixTHM_CLK_CNTL, data);
1825
1826                                 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1827                                 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1828                                         MISC_CLK_CTRL__ZCLK_SEL_MASK);
1829                                 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1830                                         (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1831                                 if (orig != data)
1832                                         WREG32_SMC(ixMISC_CLK_CTRL, data);
1833
1834                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1835                                 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1836                                 if (orig != data)
1837                                         WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1838
1839                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1840                                 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1841                                 if (orig != data)
1842                                         WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1843
1844                                 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1845                                 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1846                                 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1847                                 if (orig != data)
1848                                         WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1849                         }
1850                 }
1851         } else {
1852                 if (orig != data)
1853                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1854         }
1855
1856         orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1857         data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1858                 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1859                 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1860         if (orig != data)
1861                 WREG32_PCIE(ixPCIE_CNTL2, data);
1862
1863         if (!disable_l0s) {
1864                 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1865                 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1866                                 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1867                         data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1868                         if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1869                         (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1870                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1871                                 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1872                                 if (orig != data)
1873                                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1874                         }
1875                 }
1876         }
1877 }
1878
1879 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1880 {
1881         return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1882                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1883 }
1884
1885 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1886 {
1887         if (!ring || !ring->funcs->emit_wreg) {
1888                 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1889                 RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1890         } else {
1891                 amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1892         }
1893 }
1894
1895 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1896                                struct amdgpu_ring *ring)
1897 {
1898         if (!ring || !ring->funcs->emit_wreg) {
1899                 WREG32(mmHDP_DEBUG0, 1);
1900                 RREG32(mmHDP_DEBUG0);
1901         } else {
1902                 amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1903         }
1904 }
1905
1906 static bool cik_need_full_reset(struct amdgpu_device *adev)
1907 {
1908         /* change this when we support soft reset */
1909         return true;
1910 }
1911
1912 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1913                                uint64_t *count1)
1914 {
1915         uint32_t perfctr = 0;
1916         uint64_t cnt0_of, cnt1_of;
1917         int tmp;
1918
1919         /* This reports 0 on APUs, so return to avoid writing/reading registers
1920          * that may or may not be different from their GPU counterparts
1921          */
1922         if (adev->flags & AMD_IS_APU)
1923                 return;
1924
1925         /* Set the 2 events that we wish to watch, defined above */
1926         /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1927         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1928         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1929
1930         /* Write to enable desired perf counters */
1931         WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1932         /* Zero out and enable the perf counters
1933          * Write 0x5:
1934          * Bit 0 = Start all counters(1)
1935          * Bit 2 = Global counter reset enable(1)
1936          */
1937         WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1938
1939         msleep(1000);
1940
1941         /* Load the shadow and disable the perf counters
1942          * Write 0x2:
1943          * Bit 0 = Stop counters(0)
1944          * Bit 1 = Load the shadow counters(1)
1945          */
1946         WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1947
1948         /* Read register values to get any >32bit overflow */
1949         tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1950         cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1951         cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1952
1953         /* Get the values and add the overflow */
1954         *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1955         *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1956 }
1957
1958 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1959 {
1960         u32 clock_cntl, pc;
1961
1962         if (adev->flags & AMD_IS_APU)
1963                 return false;
1964
1965         /* check if the SMC is already running */
1966         clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1967         pc = RREG32_SMC(ixSMC_PC_C);
1968         if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1969             (0x20100 <= pc))
1970                 return true;
1971
1972         return false;
1973 }
1974
1975 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1976 {
1977         uint64_t nak_r, nak_g;
1978
1979         /* Get the number of NAKs received and generated */
1980         nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1981         nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1982
1983         /* Add the total number of NAKs, i.e the number of replays */
1984         return (nak_r + nak_g);
1985 }
1986
1987 static void cik_pre_asic_init(struct amdgpu_device *adev)
1988 {
1989 }
1990
1991 static const struct amdgpu_asic_funcs cik_asic_funcs =
1992 {
1993         .read_disabled_bios = &cik_read_disabled_bios,
1994         .read_bios_from_rom = &cik_read_bios_from_rom,
1995         .read_register = &cik_read_register,
1996         .reset = &cik_asic_reset,
1997         .reset_method = &cik_asic_reset_method,
1998         .set_vga_state = &cik_vga_set_state,
1999         .get_xclk = &cik_get_xclk,
2000         .set_uvd_clocks = &cik_set_uvd_clocks,
2001         .set_vce_clocks = &cik_set_vce_clocks,
2002         .get_config_memsize = &cik_get_config_memsize,
2003         .flush_hdp = &cik_flush_hdp,
2004         .invalidate_hdp = &cik_invalidate_hdp,
2005         .need_full_reset = &cik_need_full_reset,
2006         .init_doorbell_index = &legacy_doorbell_index_init,
2007         .get_pcie_usage = &cik_get_pcie_usage,
2008         .need_reset_on_init = &cik_need_reset_on_init,
2009         .get_pcie_replay_count = &cik_get_pcie_replay_count,
2010         .supports_baco = &cik_asic_supports_baco,
2011         .pre_asic_init = &cik_pre_asic_init,
2012         .query_video_codecs = &cik_query_video_codecs,
2013 };
2014
2015 static int cik_common_early_init(void *handle)
2016 {
2017         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2018
2019         adev->smc_rreg = &cik_smc_rreg;
2020         adev->smc_wreg = &cik_smc_wreg;
2021         adev->pcie_rreg = &cik_pcie_rreg;
2022         adev->pcie_wreg = &cik_pcie_wreg;
2023         adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2024         adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2025         adev->didt_rreg = &cik_didt_rreg;
2026         adev->didt_wreg = &cik_didt_wreg;
2027
2028         adev->asic_funcs = &cik_asic_funcs;
2029
2030         adev->rev_id = cik_get_rev_id(adev);
2031         adev->external_rev_id = 0xFF;
2032         switch (adev->asic_type) {
2033         case CHIP_BONAIRE:
2034                 adev->cg_flags =
2035                         AMD_CG_SUPPORT_GFX_MGCG |
2036                         AMD_CG_SUPPORT_GFX_MGLS |
2037                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2038                         AMD_CG_SUPPORT_GFX_CGLS |
2039                         AMD_CG_SUPPORT_GFX_CGTS |
2040                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2041                         AMD_CG_SUPPORT_GFX_CP_LS |
2042                         AMD_CG_SUPPORT_MC_LS |
2043                         AMD_CG_SUPPORT_MC_MGCG |
2044                         AMD_CG_SUPPORT_SDMA_MGCG |
2045                         AMD_CG_SUPPORT_SDMA_LS |
2046                         AMD_CG_SUPPORT_BIF_LS |
2047                         AMD_CG_SUPPORT_VCE_MGCG |
2048                         AMD_CG_SUPPORT_UVD_MGCG |
2049                         AMD_CG_SUPPORT_HDP_LS |
2050                         AMD_CG_SUPPORT_HDP_MGCG;
2051                 adev->pg_flags = 0;
2052                 adev->external_rev_id = adev->rev_id + 0x14;
2053                 break;
2054         case CHIP_HAWAII:
2055                 adev->cg_flags =
2056                         AMD_CG_SUPPORT_GFX_MGCG |
2057                         AMD_CG_SUPPORT_GFX_MGLS |
2058                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2059                         AMD_CG_SUPPORT_GFX_CGLS |
2060                         AMD_CG_SUPPORT_GFX_CGTS |
2061                         AMD_CG_SUPPORT_GFX_CP_LS |
2062                         AMD_CG_SUPPORT_MC_LS |
2063                         AMD_CG_SUPPORT_MC_MGCG |
2064                         AMD_CG_SUPPORT_SDMA_MGCG |
2065                         AMD_CG_SUPPORT_SDMA_LS |
2066                         AMD_CG_SUPPORT_BIF_LS |
2067                         AMD_CG_SUPPORT_VCE_MGCG |
2068                         AMD_CG_SUPPORT_UVD_MGCG |
2069                         AMD_CG_SUPPORT_HDP_LS |
2070                         AMD_CG_SUPPORT_HDP_MGCG;
2071                 adev->pg_flags = 0;
2072                 adev->external_rev_id = 0x28;
2073                 break;
2074         case CHIP_KAVERI:
2075                 adev->cg_flags =
2076                         AMD_CG_SUPPORT_GFX_MGCG |
2077                         AMD_CG_SUPPORT_GFX_MGLS |
2078                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2079                         AMD_CG_SUPPORT_GFX_CGLS |
2080                         AMD_CG_SUPPORT_GFX_CGTS |
2081                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2082                         AMD_CG_SUPPORT_GFX_CP_LS |
2083                         AMD_CG_SUPPORT_SDMA_MGCG |
2084                         AMD_CG_SUPPORT_SDMA_LS |
2085                         AMD_CG_SUPPORT_BIF_LS |
2086                         AMD_CG_SUPPORT_VCE_MGCG |
2087                         AMD_CG_SUPPORT_UVD_MGCG |
2088                         AMD_CG_SUPPORT_HDP_LS |
2089                         AMD_CG_SUPPORT_HDP_MGCG;
2090                 adev->pg_flags =
2091                         /*AMD_PG_SUPPORT_GFX_PG |
2092                           AMD_PG_SUPPORT_GFX_SMG |
2093                           AMD_PG_SUPPORT_GFX_DMG |*/
2094                         AMD_PG_SUPPORT_UVD |
2095                         AMD_PG_SUPPORT_VCE |
2096                         /*  AMD_PG_SUPPORT_CP |
2097                           AMD_PG_SUPPORT_GDS |
2098                           AMD_PG_SUPPORT_RLC_SMU_HS |
2099                           AMD_PG_SUPPORT_ACP |
2100                           AMD_PG_SUPPORT_SAMU |*/
2101                         0;
2102                 if (adev->pdev->device == 0x1312 ||
2103                         adev->pdev->device == 0x1316 ||
2104                         adev->pdev->device == 0x1317)
2105                         adev->external_rev_id = 0x41;
2106                 else
2107                         adev->external_rev_id = 0x1;
2108                 break;
2109         case CHIP_KABINI:
2110         case CHIP_MULLINS:
2111                 adev->cg_flags =
2112                         AMD_CG_SUPPORT_GFX_MGCG |
2113                         AMD_CG_SUPPORT_GFX_MGLS |
2114                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2115                         AMD_CG_SUPPORT_GFX_CGLS |
2116                         AMD_CG_SUPPORT_GFX_CGTS |
2117                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2118                         AMD_CG_SUPPORT_GFX_CP_LS |
2119                         AMD_CG_SUPPORT_SDMA_MGCG |
2120                         AMD_CG_SUPPORT_SDMA_LS |
2121                         AMD_CG_SUPPORT_BIF_LS |
2122                         AMD_CG_SUPPORT_VCE_MGCG |
2123                         AMD_CG_SUPPORT_UVD_MGCG |
2124                         AMD_CG_SUPPORT_HDP_LS |
2125                         AMD_CG_SUPPORT_HDP_MGCG;
2126                 adev->pg_flags =
2127                         /*AMD_PG_SUPPORT_GFX_PG |
2128                           AMD_PG_SUPPORT_GFX_SMG | */
2129                         AMD_PG_SUPPORT_UVD |
2130                         /*AMD_PG_SUPPORT_VCE |
2131                           AMD_PG_SUPPORT_CP |
2132                           AMD_PG_SUPPORT_GDS |
2133                           AMD_PG_SUPPORT_RLC_SMU_HS |
2134                           AMD_PG_SUPPORT_SAMU |*/
2135                         0;
2136                 if (adev->asic_type == CHIP_KABINI) {
2137                         if (adev->rev_id == 0)
2138                                 adev->external_rev_id = 0x81;
2139                         else if (adev->rev_id == 1)
2140                                 adev->external_rev_id = 0x82;
2141                         else if (adev->rev_id == 2)
2142                                 adev->external_rev_id = 0x85;
2143                 } else
2144                         adev->external_rev_id = adev->rev_id + 0xa1;
2145                 break;
2146         default:
2147                 /* FIXME: not supported yet */
2148                 return -EINVAL;
2149         }
2150
2151         return 0;
2152 }
2153
2154 static int cik_common_sw_init(void *handle)
2155 {
2156         return 0;
2157 }
2158
2159 static int cik_common_sw_fini(void *handle)
2160 {
2161         return 0;
2162 }
2163
2164 static int cik_common_hw_init(void *handle)
2165 {
2166         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2167
2168         /* move the golden regs per IP block */
2169         cik_init_golden_registers(adev);
2170         /* enable pcie gen2/3 link */
2171         cik_pcie_gen3_enable(adev);
2172         /* enable aspm */
2173         cik_program_aspm(adev);
2174
2175         return 0;
2176 }
2177
2178 static int cik_common_hw_fini(void *handle)
2179 {
2180         return 0;
2181 }
2182
2183 static int cik_common_suspend(void *handle)
2184 {
2185         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2186
2187         return cik_common_hw_fini(adev);
2188 }
2189
2190 static int cik_common_resume(void *handle)
2191 {
2192         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2193
2194         return cik_common_hw_init(adev);
2195 }
2196
2197 static bool cik_common_is_idle(void *handle)
2198 {
2199         return true;
2200 }
2201
2202 static int cik_common_wait_for_idle(void *handle)
2203 {
2204         return 0;
2205 }
2206
2207 static int cik_common_soft_reset(void *handle)
2208 {
2209         /* XXX hard reset?? */
2210         return 0;
2211 }
2212
2213 static int cik_common_set_clockgating_state(void *handle,
2214                                             enum amd_clockgating_state state)
2215 {
2216         return 0;
2217 }
2218
2219 static int cik_common_set_powergating_state(void *handle,
2220                                             enum amd_powergating_state state)
2221 {
2222         return 0;
2223 }
2224
2225 static const struct amd_ip_funcs cik_common_ip_funcs = {
2226         .name = "cik_common",
2227         .early_init = cik_common_early_init,
2228         .late_init = NULL,
2229         .sw_init = cik_common_sw_init,
2230         .sw_fini = cik_common_sw_fini,
2231         .hw_init = cik_common_hw_init,
2232         .hw_fini = cik_common_hw_fini,
2233         .suspend = cik_common_suspend,
2234         .resume = cik_common_resume,
2235         .is_idle = cik_common_is_idle,
2236         .wait_for_idle = cik_common_wait_for_idle,
2237         .soft_reset = cik_common_soft_reset,
2238         .set_clockgating_state = cik_common_set_clockgating_state,
2239         .set_powergating_state = cik_common_set_powergating_state,
2240 };
2241
2242 static const struct amdgpu_ip_block_version cik_common_ip_block =
2243 {
2244         .type = AMD_IP_BLOCK_TYPE_COMMON,
2245         .major = 1,
2246         .minor = 0,
2247         .rev = 0,
2248         .funcs = &cik_common_ip_funcs,
2249 };
2250
2251 int cik_set_ip_blocks(struct amdgpu_device *adev)
2252 {
2253         switch (adev->asic_type) {
2254         case CHIP_BONAIRE:
2255                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2256                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2257                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2258                 amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2259                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2260                 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2261                 if (adev->enable_virtual_display)
2262                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2263 #if defined(CONFIG_DRM_AMD_DC)
2264                 else if (amdgpu_device_has_dc_support(adev))
2265                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2266 #endif
2267                 else
2268                         amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2269                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2270                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2271                 break;
2272         case CHIP_HAWAII:
2273                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2274                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2275                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2276                 amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2277                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2278                 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2279                 if (adev->enable_virtual_display)
2280                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2281 #if defined(CONFIG_DRM_AMD_DC)
2282                 else if (amdgpu_device_has_dc_support(adev))
2283                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2284 #endif
2285                 else
2286                         amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2287                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2288                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2289                 break;
2290         case CHIP_KAVERI:
2291                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2292                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2293                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2294                 amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2295                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2296                 amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2297                 if (adev->enable_virtual_display)
2298                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2299 #if defined(CONFIG_DRM_AMD_DC)
2300                 else if (amdgpu_device_has_dc_support(adev))
2301                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2302 #endif
2303                 else
2304                         amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2305
2306                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2307                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2308                 break;
2309         case CHIP_KABINI:
2310         case CHIP_MULLINS:
2311                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2312                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2313                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2314                 amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2315                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2316                 amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2317                 if (adev->enable_virtual_display)
2318                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2319 #if defined(CONFIG_DRM_AMD_DC)
2320                 else if (amdgpu_device_has_dc_support(adev))
2321                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2322 #endif
2323                 else
2324                         amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2325                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2326                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2327                 break;
2328         default:
2329                 /* FIXME: not supported yet */
2330                 return -EINVAL;
2331         }
2332         return 0;
2333 }