drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 /*
  40  * Indirect registers accessor
  41  */
  42 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  43 {
  44         unsigned long flags;
  45         u32 r;
  46
  47         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  48         WREG32(TN_SMC_IND_INDEX_0, (reg));
  49         r = RREG32(TN_SMC_IND_DATA_0);
  50         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  51         return r;
  52 }
  53
  54 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  55 {
  56         unsigned long flags;
  57
  58         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  59         WREG32(TN_SMC_IND_INDEX_0, (reg));
  60         WREG32(TN_SMC_IND_DATA_0, (v));
  61         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  62 }
  63
  64 static const u32 tn_rlc_save_restore_register_list[] =
  65 {
  66         0x98fc,
  67         0x98f0,
  68         0x9834,
  69         0x9838,
  70         0x9870,
  71         0x9874,
  72         0x8a14,
  73         0x8b24,
  74         0x8bcc,
  75         0x8b10,
  76         0x8c30,
  77         0x8d00,
  78         0x8d04,
  79         0x8c00,
  80         0x8c04,
  81         0x8c10,
  82         0x8c14,
  83         0x8d8c,
  84         0x8cf0,
  85         0x8e38,
  86         0x9508,
  87         0x9688,
  88         0x9608,
  89         0x960c,
  90         0x9610,
  91         0x9614,
  92         0x88c4,
  93         0x8978,
  94         0x88d4,
  95         0x900c,
  96         0x9100,
  97         0x913c,
  98         0x90e8,
  99         0x9354,
 100         0xa008,
 101         0x98f8,
 102         0x9148,
 103         0x914c,
 104         0x3f94,
 105         0x98f4,
 106         0x9b7c,
 107         0x3f8c,
 108         0x8950,
 109         0x8954,
 110         0x8a18,
 111         0x8b28,
 112         0x9144,
 113         0x3f90,
 114         0x915c,
 115         0x9160,
 116         0x9178,
 117         0x917c,
 118         0x9180,
 119         0x918c,
 120         0x9190,
 121         0x9194,
 122         0x9198,
 123         0x919c,
 124         0x91a8,
 125         0x91ac,
 126         0x91b0,
 127         0x91b4,
 128         0x91b8,
 129         0x91c4,
 130         0x91c8,
 131         0x91cc,
 132         0x91d0,
 133         0x91d4,
 134         0x91e0,
 135         0x91e4,
 136         0x91ec,
 137         0x91f0,
 138         0x91f4,
 139         0x9200,
 140         0x9204,
 141         0x929c,
 142         0x8030,
 143         0x9150,
 144         0x9a60,
 145         0x920c,
 146         0x9210,
 147         0x9228,
 148         0x922c,
 149         0x9244,
 150         0x9248,
 151         0x91e8,
 152         0x9294,
 153         0x9208,
 154         0x9224,
 155         0x9240,
 156         0x9220,
 157         0x923c,
 158         0x9258,
 159         0x9744,
 160         0xa200,
 161         0xa204,
 162         0xa208,
 163         0xa20c,
 164         0x8d58,
 165         0x9030,
 166         0x9034,
 167         0x9038,
 168         0x903c,
 169         0x9040,
 170         0x9654,
 171         0x897c,
 172         0xa210,
 173         0xa214,
 174         0x9868,
 175         0xa02c,
 176         0x9664,
 177         0x9698,
 178         0x949c,
 179         0x8e10,
 180         0x8e18,
 181         0x8c50,
 182         0x8c58,
 183         0x8c60,
 184         0x8c68,
 185         0x89b4,
 186         0x9830,
 187         0x802c,
 188 };
 189
 190 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 191 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 192 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 193 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 194 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 195 extern void evergreen_mc_program(struct radeon_device *rdev);
 196 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 197 extern int evergreen_mc_init(struct radeon_device *rdev);
 198 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 199 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 200 extern void evergreen_program_aspm(struct radeon_device *rdev);
 201 extern void sumo_rlc_fini(struct radeon_device *rdev);
 202 extern int sumo_rlc_init(struct radeon_device *rdev);
 203 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 204
 205 /* Firmware Names */
 206 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 207 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 208 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 209 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 210 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 211 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 212 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 213 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 214 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 215 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 216 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 217 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 218 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 219 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 220 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 221 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 222 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 223 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 224 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 225 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 226 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 227
 228
 229 static const u32 cayman_golden_registers2[] =
 230 {
 231         0x3e5c, 0xffffffff, 0x00000000,
 232         0x3e48, 0xffffffff, 0x00000000,
 233         0x3e4c, 0xffffffff, 0x00000000,
 234         0x3e64, 0xffffffff, 0x00000000,
 235         0x3e50, 0xffffffff, 0x00000000,
 236         0x3e60, 0xffffffff, 0x00000000
 237 };
 238
 239 static const u32 cayman_golden_registers[] =
 240 {
 241         0x5eb4, 0xffffffff, 0x00000002,
 242         0x5e78, 0x8f311ff1, 0x001000f0,
 243         0x3f90, 0xffff0000, 0xff000000,
 244         0x9148, 0xffff0000, 0xff000000,
 245         0x3f94, 0xffff0000, 0xff000000,
 246         0x914c, 0xffff0000, 0xff000000,
 247         0xc78, 0x00000080, 0x00000080,
 248         0xbd4, 0x70073777, 0x00011003,
 249         0xd02c, 0xbfffff1f, 0x08421000,
 250         0xd0b8, 0x73773777, 0x02011003,
 251         0x5bc0, 0x00200000, 0x50100000,
 252         0x98f8, 0x33773777, 0x02011003,
 253         0x98fc, 0xffffffff, 0x76541032,
 254         0x7030, 0x31000311, 0x00000011,
 255         0x2f48, 0x33773777, 0x42010001,
 256         0x6b28, 0x00000010, 0x00000012,
 257         0x7728, 0x00000010, 0x00000012,
 258         0x10328, 0x00000010, 0x00000012,
 259         0x10f28, 0x00000010, 0x00000012,
 260         0x11b28, 0x00000010, 0x00000012,
 261         0x12728, 0x00000010, 0x00000012,
 262         0x240c, 0x000007ff, 0x00000000,
 263         0x8a14, 0xf000001f, 0x00000007,
 264         0x8b24, 0x3fff3fff, 0x00ff0fff,
 265         0x8b10, 0x0000ff0f, 0x00000000,
 266         0x28a4c, 0x07ffffff, 0x06000000,
 267         0x10c, 0x00000001, 0x00010003,
 268         0xa02c, 0xffffffff, 0x0000009b,
 269         0x913c, 0x0000010f, 0x01000100,
 270         0x8c04, 0xf8ff00ff, 0x40600060,
 271         0x28350, 0x00000f01, 0x00000000,
 272         0x9508, 0x3700001f, 0x00000002,
 273         0x960c, 0xffffffff, 0x54763210,
 274         0x88c4, 0x001f3ae3, 0x00000082,
 275         0x88d0, 0xffffffff, 0x0f40df40,
 276         0x88d4, 0x0000001f, 0x00000010,
 277         0x8974, 0xffffffff, 0x00000000
 278 };
 279
 280 static const u32 dvst_golden_registers2[] =
 281 {
 282         0x8f8, 0xffffffff, 0,
 283         0x8fc, 0x00380000, 0,
 284         0x8f8, 0xffffffff, 1,
 285         0x8fc, 0x0e000000, 0
 286 };
 287
 288 static const u32 dvst_golden_registers[] =
 289 {
 290         0x690, 0x3fff3fff, 0x20c00033,
 291         0x918c, 0x0fff0fff, 0x00010006,
 292         0x91a8, 0x0fff0fff, 0x00010006,
 293         0x9150, 0xffffdfff, 0x6e944040,
 294         0x917c, 0x0fff0fff, 0x00030002,
 295         0x9198, 0x0fff0fff, 0x00030002,
 296         0x915c, 0x0fff0fff, 0x00010000,
 297         0x3f90, 0xffff0001, 0xff000000,
 298         0x9178, 0x0fff0fff, 0x00070000,
 299         0x9194, 0x0fff0fff, 0x00070000,
 300         0x9148, 0xffff0001, 0xff000000,
 301         0x9190, 0x0fff0fff, 0x00090008,
 302         0x91ac, 0x0fff0fff, 0x00090008,
 303         0x3f94, 0xffff0000, 0xff000000,
 304         0x914c, 0xffff0000, 0xff000000,
 305         0x929c, 0x00000fff, 0x00000001,
 306         0x55e4, 0xff607fff, 0xfc000100,
 307         0x8a18, 0xff000fff, 0x00000100,
 308         0x8b28, 0xff000fff, 0x00000100,
 309         0x9144, 0xfffc0fff, 0x00000100,
 310         0x6ed8, 0x00010101, 0x00010000,
 311         0x9830, 0xffffffff, 0x00000000,
 312         0x9834, 0xf00fffff, 0x00000400,
 313         0x9838, 0xfffffffe, 0x00000000,
 314         0xd0c0, 0xff000fff, 0x00000100,
 315         0xd02c, 0xbfffff1f, 0x08421000,
 316         0xd0b8, 0x73773777, 0x12010001,
 317         0x5bb0, 0x000000f0, 0x00000070,
 318         0x98f8, 0x73773777, 0x12010001,
 319         0x98fc, 0xffffffff, 0x00000010,
 320         0x9b7c, 0x00ff0000, 0x00fc0000,
 321         0x8030, 0x00001f0f, 0x0000100a,
 322         0x2f48, 0x73773777, 0x12010001,
 323         0x2408, 0x00030000, 0x000c007f,
 324         0x8a14, 0xf000003f, 0x00000007,
 325         0x8b24, 0x3fff3fff, 0x00ff0fff,
 326         0x8b10, 0x0000ff0f, 0x00000000,
 327         0x28a4c, 0x07ffffff, 0x06000000,
 328         0x4d8, 0x00000fff, 0x00000100,
 329         0xa008, 0xffffffff, 0x00010000,
 330         0x913c, 0xffff03ff, 0x01000100,
 331         0x8c00, 0x000000ff, 0x00000003,
 332         0x8c04, 0xf8ff00ff, 0x40600060,
 333         0x8cf0, 0x1fff1fff, 0x08e00410,
 334         0x28350, 0x00000f01, 0x00000000,
 335         0x9508, 0xf700071f, 0x00000002,
 336         0x960c, 0xffffffff, 0x54763210,
 337         0x20ef8, 0x01ff01ff, 0x00000002,
 338         0x20e98, 0xfffffbff, 0x00200000,
 339         0x2015c, 0xffffffff, 0x00000f40,
 340         0x88c4, 0x001f3ae3, 0x00000082,
 341         0x8978, 0x3fffffff, 0x04050140,
 342         0x88d4, 0x0000001f, 0x00000010,
 343         0x8974, 0xffffffff, 0x00000000
 344 };
 345
 346 static const u32 scrapper_golden_registers[] =
 347 {
 348         0x690, 0x3fff3fff, 0x20c00033,
 349         0x918c, 0x0fff0fff, 0x00010006,
 350         0x918c, 0x0fff0fff, 0x00010006,
 351         0x91a8, 0x0fff0fff, 0x00010006,
 352         0x91a8, 0x0fff0fff, 0x00010006,
 353         0x9150, 0xffffdfff, 0x6e944040,
 354         0x9150, 0xffffdfff, 0x6e944040,
 355         0x917c, 0x0fff0fff, 0x00030002,
 356         0x917c, 0x0fff0fff, 0x00030002,
 357         0x9198, 0x0fff0fff, 0x00030002,
 358         0x9198, 0x0fff0fff, 0x00030002,
 359         0x915c, 0x0fff0fff, 0x00010000,
 360         0x915c, 0x0fff0fff, 0x00010000,
 361         0x3f90, 0xffff0001, 0xff000000,
 362         0x3f90, 0xffff0001, 0xff000000,
 363         0x9178, 0x0fff0fff, 0x00070000,
 364         0x9178, 0x0fff0fff, 0x00070000,
 365         0x9194, 0x0fff0fff, 0x00070000,
 366         0x9194, 0x0fff0fff, 0x00070000,
 367         0x9148, 0xffff0001, 0xff000000,
 368         0x9148, 0xffff0001, 0xff000000,
 369         0x9190, 0x0fff0fff, 0x00090008,
 370         0x9190, 0x0fff0fff, 0x00090008,
 371         0x91ac, 0x0fff0fff, 0x00090008,
 372         0x91ac, 0x0fff0fff, 0x00090008,
 373         0x3f94, 0xffff0000, 0xff000000,
 374         0x3f94, 0xffff0000, 0xff000000,
 375         0x914c, 0xffff0000, 0xff000000,
 376         0x914c, 0xffff0000, 0xff000000,
 377         0x929c, 0x00000fff, 0x00000001,
 378         0x929c, 0x00000fff, 0x00000001,
 379         0x55e4, 0xff607fff, 0xfc000100,
 380         0x8a18, 0xff000fff, 0x00000100,
 381         0x8a18, 0xff000fff, 0x00000100,
 382         0x8b28, 0xff000fff, 0x00000100,
 383         0x8b28, 0xff000fff, 0x00000100,
 384         0x9144, 0xfffc0fff, 0x00000100,
 385         0x9144, 0xfffc0fff, 0x00000100,
 386         0x6ed8, 0x00010101, 0x00010000,
 387         0x9830, 0xffffffff, 0x00000000,
 388         0x9830, 0xffffffff, 0x00000000,
 389         0x9834, 0xf00fffff, 0x00000400,
 390         0x9834, 0xf00fffff, 0x00000400,
 391         0x9838, 0xfffffffe, 0x00000000,
 392         0x9838, 0xfffffffe, 0x00000000,
 393         0xd0c0, 0xff000fff, 0x00000100,
 394         0xd02c, 0xbfffff1f, 0x08421000,
 395         0xd02c, 0xbfffff1f, 0x08421000,
 396         0xd0b8, 0x73773777, 0x12010001,
 397         0xd0b8, 0x73773777, 0x12010001,
 398         0x5bb0, 0x000000f0, 0x00000070,
 399         0x98f8, 0x73773777, 0x12010001,
 400         0x98f8, 0x73773777, 0x12010001,
 401         0x98fc, 0xffffffff, 0x00000010,
 402         0x98fc, 0xffffffff, 0x00000010,
 403         0x9b7c, 0x00ff0000, 0x00fc0000,
 404         0x9b7c, 0x00ff0000, 0x00fc0000,
 405         0x8030, 0x00001f0f, 0x0000100a,
 406         0x8030, 0x00001f0f, 0x0000100a,
 407         0x2f48, 0x73773777, 0x12010001,
 408         0x2f48, 0x73773777, 0x12010001,
 409         0x2408, 0x00030000, 0x000c007f,
 410         0x8a14, 0xf000003f, 0x00000007,
 411         0x8a14, 0xf000003f, 0x00000007,
 412         0x8b24, 0x3fff3fff, 0x00ff0fff,
 413         0x8b24, 0x3fff3fff, 0x00ff0fff,
 414         0x8b10, 0x0000ff0f, 0x00000000,
 415         0x8b10, 0x0000ff0f, 0x00000000,
 416         0x28a4c, 0x07ffffff, 0x06000000,
 417         0x28a4c, 0x07ffffff, 0x06000000,
 418         0x4d8, 0x00000fff, 0x00000100,
 419         0x4d8, 0x00000fff, 0x00000100,
 420         0xa008, 0xffffffff, 0x00010000,
 421         0xa008, 0xffffffff, 0x00010000,
 422         0x913c, 0xffff03ff, 0x01000100,
 423         0x913c, 0xffff03ff, 0x01000100,
 424         0x90e8, 0x001fffff, 0x010400c0,
 425         0x8c00, 0x000000ff, 0x00000003,
 426         0x8c00, 0x000000ff, 0x00000003,
 427         0x8c04, 0xf8ff00ff, 0x40600060,
 428         0x8c04, 0xf8ff00ff, 0x40600060,
 429         0x8c30, 0x0000000f, 0x00040005,
 430         0x8cf0, 0x1fff1fff, 0x08e00410,
 431         0x8cf0, 0x1fff1fff, 0x08e00410,
 432         0x900c, 0x00ffffff, 0x0017071f,
 433         0x28350, 0x00000f01, 0x00000000,
 434         0x28350, 0x00000f01, 0x00000000,
 435         0x9508, 0xf700071f, 0x00000002,
 436         0x9508, 0xf700071f, 0x00000002,
 437         0x9688, 0x00300000, 0x0017000f,
 438         0x960c, 0xffffffff, 0x54763210,
 439         0x960c, 0xffffffff, 0x54763210,
 440         0x20ef8, 0x01ff01ff, 0x00000002,
 441         0x20e98, 0xfffffbff, 0x00200000,
 442         0x2015c, 0xffffffff, 0x00000f40,
 443         0x88c4, 0x001f3ae3, 0x00000082,
 444         0x88c4, 0x001f3ae3, 0x00000082,
 445         0x8978, 0x3fffffff, 0x04050140,
 446         0x8978, 0x3fffffff, 0x04050140,
 447         0x88d4, 0x0000001f, 0x00000010,
 448         0x88d4, 0x0000001f, 0x00000010,
 449         0x8974, 0xffffffff, 0x00000000,
 450         0x8974, 0xffffffff, 0x00000000
 451 };
 452
 453 static void ni_init_golden_registers(struct radeon_device *rdev)
 454 {
 455         switch (rdev->family) {
 456         case CHIP_CAYMAN:
 457                 radeon_program_register_sequence(rdev,
 458                                                  cayman_golden_registers,
 459                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 460                 radeon_program_register_sequence(rdev,
 461                                                  cayman_golden_registers2,
 462                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 463                 break;
 464         case CHIP_ARUBA:
 465                 if ((rdev->pdev->device == 0x9900) ||
 466                     (rdev->pdev->device == 0x9901) ||
 467                     (rdev->pdev->device == 0x9903) ||
 468                     (rdev->pdev->device == 0x9904) ||
 469                     (rdev->pdev->device == 0x9905) ||
 470                     (rdev->pdev->device == 0x9906) ||
 471                     (rdev->pdev->device == 0x9907) ||
 472                     (rdev->pdev->device == 0x9908) ||
 473                     (rdev->pdev->device == 0x9909) ||
 474                     (rdev->pdev->device == 0x990A) ||
 475                     (rdev->pdev->device == 0x990B) ||
 476                     (rdev->pdev->device == 0x990C) ||
 477                     (rdev->pdev->device == 0x990D) ||
 478                     (rdev->pdev->device == 0x990E) ||
 479                     (rdev->pdev->device == 0x990F) ||
 480                     (rdev->pdev->device == 0x9910) ||
 481                     (rdev->pdev->device == 0x9913) ||
 482                     (rdev->pdev->device == 0x9917) ||
 483                     (rdev->pdev->device == 0x9918)) {
 484                         radeon_program_register_sequence(rdev,
 485                                                          dvst_golden_registers,
 486                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 487                         radeon_program_register_sequence(rdev,
 488                                                          dvst_golden_registers2,
 489                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 490                 } else {
 491                         radeon_program_register_sequence(rdev,
 492                                                          scrapper_golden_registers,
 493                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 494                         radeon_program_register_sequence(rdev,
 495                                                          dvst_golden_registers2,
 496                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 497                 }
 498                 break;
 499         default:
 500                 break;
 501         }
 502 }
 503
 504 #define BTC_IO_MC_REGS_SIZE 29
 505
 506 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 507         {0x00000077, 0xff010100},
 508         {0x00000078, 0x00000000},
 509         {0x00000079, 0x00001434},
 510         {0x0000007a, 0xcc08ec08},
 511         {0x0000007b, 0x00040000},
 512         {0x0000007c, 0x000080c0},
 513         {0x0000007d, 0x09000000},
 514         {0x0000007e, 0x00210404},
 515         {0x00000081, 0x08a8e800},
 516         {0x00000082, 0x00030444},
 517         {0x00000083, 0x00000000},
 518         {0x00000085, 0x00000001},
 519         {0x00000086, 0x00000002},
 520         {0x00000087, 0x48490000},
 521         {0x00000088, 0x20244647},
 522         {0x00000089, 0x00000005},
 523         {0x0000008b, 0x66030000},
 524         {0x0000008c, 0x00006603},
 525         {0x0000008d, 0x00000100},
 526         {0x0000008f, 0x00001c0a},
 527         {0x00000090, 0xff000001},
 528         {0x00000094, 0x00101101},
 529         {0x00000095, 0x00000fff},
 530         {0x00000096, 0x00116fff},
 531         {0x00000097, 0x60010000},
 532         {0x00000098, 0x10010000},
 533         {0x00000099, 0x00006000},
 534         {0x0000009a, 0x00001000},
 535         {0x0000009f, 0x00946a00}
 536 };
 537
 538 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 539         {0x00000077, 0xff010100},
 540         {0x00000078, 0x00000000},
 541         {0x00000079, 0x00001434},
 542         {0x0000007a, 0xcc08ec08},
 543         {0x0000007b, 0x00040000},
 544         {0x0000007c, 0x000080c0},
 545         {0x0000007d, 0x09000000},
 546         {0x0000007e, 0x00210404},
 547         {0x00000081, 0x08a8e800},
 548         {0x00000082, 0x00030444},
 549         {0x00000083, 0x00000000},
 550         {0x00000085, 0x00000001},
 551         {0x00000086, 0x00000002},
 552         {0x00000087, 0x48490000},
 553         {0x00000088, 0x20244647},
 554         {0x00000089, 0x00000005},
 555         {0x0000008b, 0x66030000},
 556         {0x0000008c, 0x00006603},
 557         {0x0000008d, 0x00000100},
 558         {0x0000008f, 0x00001c0a},
 559         {0x00000090, 0xff000001},
 560         {0x00000094, 0x00101101},
 561         {0x00000095, 0x00000fff},
 562         {0x00000096, 0x00116fff},
 563         {0x00000097, 0x60010000},
 564         {0x00000098, 0x10010000},
 565         {0x00000099, 0x00006000},
 566         {0x0000009a, 0x00001000},
 567         {0x0000009f, 0x00936a00}
 568 };
 569
 570 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 571         {0x00000077, 0xff010100},
 572         {0x00000078, 0x00000000},
 573         {0x00000079, 0x00001434},
 574         {0x0000007a, 0xcc08ec08},
 575         {0x0000007b, 0x00040000},
 576         {0x0000007c, 0x000080c0},
 577         {0x0000007d, 0x09000000},
 578         {0x0000007e, 0x00210404},
 579         {0x00000081, 0x08a8e800},
 580         {0x00000082, 0x00030444},
 581         {0x00000083, 0x00000000},
 582         {0x00000085, 0x00000001},
 583         {0x00000086, 0x00000002},
 584         {0x00000087, 0x48490000},
 585         {0x00000088, 0x20244647},
 586         {0x00000089, 0x00000005},
 587         {0x0000008b, 0x66030000},
 588         {0x0000008c, 0x00006603},
 589         {0x0000008d, 0x00000100},
 590         {0x0000008f, 0x00001c0a},
 591         {0x00000090, 0xff000001},
 592         {0x00000094, 0x00101101},
 593         {0x00000095, 0x00000fff},
 594         {0x00000096, 0x00116fff},
 595         {0x00000097, 0x60010000},
 596         {0x00000098, 0x10010000},
 597         {0x00000099, 0x00006000},
 598         {0x0000009a, 0x00001000},
 599         {0x0000009f, 0x00916a00}
 600 };
 601
 602 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 603         {0x00000077, 0xff010100},
 604         {0x00000078, 0x00000000},
 605         {0x00000079, 0x00001434},
 606         {0x0000007a, 0xcc08ec08},
 607         {0x0000007b, 0x00040000},
 608         {0x0000007c, 0x000080c0},
 609         {0x0000007d, 0x09000000},
 610         {0x0000007e, 0x00210404},
 611         {0x00000081, 0x08a8e800},
 612         {0x00000082, 0x00030444},
 613         {0x00000083, 0x00000000},
 614         {0x00000085, 0x00000001},
 615         {0x00000086, 0x00000002},
 616         {0x00000087, 0x48490000},
 617         {0x00000088, 0x20244647},
 618         {0x00000089, 0x00000005},
 619         {0x0000008b, 0x66030000},
 620         {0x0000008c, 0x00006603},
 621         {0x0000008d, 0x00000100},
 622         {0x0000008f, 0x00001c0a},
 623         {0x00000090, 0xff000001},
 624         {0x00000094, 0x00101101},
 625         {0x00000095, 0x00000fff},
 626         {0x00000096, 0x00116fff},
 627         {0x00000097, 0x60010000},
 628         {0x00000098, 0x10010000},
 629         {0x00000099, 0x00006000},
 630         {0x0000009a, 0x00001000},
 631         {0x0000009f, 0x00976b00}
 632 };
 633
 634 int ni_mc_load_microcode(struct radeon_device *rdev)
 635 {
 636         const __be32 *fw_data;
 637         u32 mem_type, running, blackout = 0;
 638         u32 *io_mc_regs;
 639         int i, ucode_size, regs_size;
 640
 641         if (!rdev->mc_fw)
 642                 return -EINVAL;
 643
 644         switch (rdev->family) {
 645         case CHIP_BARTS:
 646                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 647                 ucode_size = BTC_MC_UCODE_SIZE;
 648                 regs_size = BTC_IO_MC_REGS_SIZE;
 649                 break;
 650         case CHIP_TURKS:
 651                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 652                 ucode_size = BTC_MC_UCODE_SIZE;
 653                 regs_size = BTC_IO_MC_REGS_SIZE;
 654                 break;
 655         case CHIP_CAICOS:
 656         default:
 657                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 658                 ucode_size = BTC_MC_UCODE_SIZE;
 659                 regs_size = BTC_IO_MC_REGS_SIZE;
 660                 break;
 661         case CHIP_CAYMAN:
 662                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 663                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 664                 regs_size = BTC_IO_MC_REGS_SIZE;
 665                 break;
 666         }
 667
 668         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 669         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 670
 671         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 672                 if (running) {
 673                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 674                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 675                 }
 676
 677                 /* reset the engine and set to writable */
 678                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 679                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 680
 681                 /* load mc io regs */
 682                 for (i = 0; i < regs_size; i++) {
 683                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 684                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 685                 }
 686                 /* load the MC ucode */
 687                 fw_data = (const __be32 *)rdev->mc_fw->data;
 688                 for (i = 0; i < ucode_size; i++)
 689                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 690
 691                 /* put the engine back into the active state */
 692                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 693                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 694                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 695
 696                 /* wait for training to complete */
 697                 for (i = 0; i < rdev->usec_timeout; i++) {
 698                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 699                                 break;
 700                         udelay(1);
 701                 }
 702
 703                 if (running)
 704                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 705         }
 706
 707         return 0;
 708 }
 709
 710 int ni_init_microcode(struct radeon_device *rdev)
 711 {
 712         const char *chip_name;
 713         const char *rlc_chip_name;
 714         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 715         size_t smc_req_size = 0;
 716         char fw_name[30];
 717         int err;
 718
 719         DRM_DEBUG("\n");
 720
 721         switch (rdev->family) {
 722         case CHIP_BARTS:
 723                 chip_name = "BARTS";
 724                 rlc_chip_name = "BTC";
 725                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 726                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 727                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 728                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 729                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 730                 break;
 731         case CHIP_TURKS:
 732                 chip_name = "TURKS";
 733                 rlc_chip_name = "BTC";
 734                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 735                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 736                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 737                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 738                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 739                 break;
 740         case CHIP_CAICOS:
 741                 chip_name = "CAICOS";
 742                 rlc_chip_name = "BTC";
 743                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 744                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 745                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 746                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 747                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 748                 break;
 749         case CHIP_CAYMAN:
 750                 chip_name = "CAYMAN";
 751                 rlc_chip_name = "CAYMAN";
 752                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 753                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 754                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 755                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 756                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 757                 break;
 758         case CHIP_ARUBA:
 759                 chip_name = "ARUBA";
 760                 rlc_chip_name = "ARUBA";
 761                 /* pfp/me same size as CAYMAN */
 762                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 763                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 764                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 765                 mc_req_size = 0;
 766                 break;
 767         default: BUG();
 768         }
 769
 770         DRM_INFO("Loading %s Microcode\n", chip_name);
 771
 772         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 773         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 774         if (err)
 775                 goto out;
 776         if (rdev->pfp_fw->size != pfp_req_size) {
 777                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 778                        rdev->pfp_fw->size, fw_name);
 779                 err = -EINVAL;
 780                 goto out;
 781         }
 782
 783         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 784         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 785         if (err)
 786                 goto out;
 787         if (rdev->me_fw->size != me_req_size) {
 788                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 789                        rdev->me_fw->size, fw_name);
 790                 err = -EINVAL;
 791         }
 792
 793         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 794         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 795         if (err)
 796                 goto out;
 797         if (rdev->rlc_fw->size != rlc_req_size) {
 798                 pr_err("ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 799                        rdev->rlc_fw->size, fw_name);
 800                 err = -EINVAL;
 801         }
 802
 803         /* no MC ucode on TN */
 804         if (!(rdev->flags & RADEON_IS_IGP)) {
 805                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 806                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 807                 if (err)
 808                         goto out;
 809                 if (rdev->mc_fw->size != mc_req_size) {
 810                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 811                                rdev->mc_fw->size, fw_name);
 812                         err = -EINVAL;
 813                 }
 814         }
 815
 816         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 817                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 818                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 819                 if (err) {
 820                         pr_err("smc: error loading firmware \"%s\"\n", fw_name);
 821                         release_firmware(rdev->smc_fw);
 822                         rdev->smc_fw = NULL;
 823                         err = 0;
 824                 } else if (rdev->smc_fw->size != smc_req_size) {
 825                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 826                                rdev->mc_fw->size, fw_name);
 827                         err = -EINVAL;
 828                 }
 829         }
 830
 831 out:
 832         if (err) {
 833                 if (err != -EINVAL)
 834                         pr_err("ni_cp: Failed to load firmware \"%s\"\n",
 835                                fw_name);
 836                 release_firmware(rdev->pfp_fw);
 837                 rdev->pfp_fw = NULL;
 838                 release_firmware(rdev->me_fw);
 839                 rdev->me_fw = NULL;
 840                 release_firmware(rdev->rlc_fw);
 841                 rdev->rlc_fw = NULL;
 842                 release_firmware(rdev->mc_fw);
 843                 rdev->mc_fw = NULL;
 844         }
 845         return err;
 846 }
 847
 848 /**
 849  * cayman_get_allowed_info_register - fetch the register for the info ioctl
 850  *
 851  * @rdev: radeon_device pointer
 852  * @reg: register offset in bytes
 853  * @val: register value
 854  *
 855  * Returns 0 for success or -EINVAL for an invalid register
 856  *
 857  */
 858 int cayman_get_allowed_info_register(struct radeon_device *rdev,
 859                                      u32 reg, u32 *val)
 860 {
 861         switch (reg) {
 862         case GRBM_STATUS:
 863         case GRBM_STATUS_SE0:
 864         case GRBM_STATUS_SE1:
 865         case SRBM_STATUS:
 866         case SRBM_STATUS2:
 867         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 868         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 869         case UVD_STATUS:
 870                 *val = RREG32(reg);
 871                 return 0;
 872         default:
 873                 return -EINVAL;
 874         }
 875 }
 876
 877 int tn_get_temp(struct radeon_device *rdev)
 878 {
 879         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 880         int actual_temp = (temp / 8) - 49;
 881
 882         return actual_temp * 1000;
 883 }
 884
 885 /*
 886  * Core functions
 887  */
 888 static void cayman_gpu_init(struct radeon_device *rdev)
 889 {
 890         u32 gb_addr_config = 0;
 891         u32 mc_shared_chmap, mc_arb_ramcfg;
 892         u32 cgts_tcc_disable;
 893         u32 sx_debug_1;
 894         u32 smx_dc_ctl0;
 895         u32 cgts_sm_ctrl_reg;
 896         u32 hdp_host_path_cntl;
 897         u32 tmp;
 898         u32 disabled_rb_mask;
 899         int i, j;
 900
 901         switch (rdev->family) {
 902         case CHIP_CAYMAN:
 903                 rdev->config.cayman.max_shader_engines = 2;
 904                 rdev->config.cayman.max_pipes_per_simd = 4;
 905                 rdev->config.cayman.max_tile_pipes = 8;
 906                 rdev->config.cayman.max_simds_per_se = 12;
 907                 rdev->config.cayman.max_backends_per_se = 4;
 908                 rdev->config.cayman.max_texture_channel_caches = 8;
 909                 rdev->config.cayman.max_gprs = 256;
 910                 rdev->config.cayman.max_threads = 256;
 911                 rdev->config.cayman.max_gs_threads = 32;
 912                 rdev->config.cayman.max_stack_entries = 512;
 913                 rdev->config.cayman.sx_num_of_sets = 8;
 914                 rdev->config.cayman.sx_max_export_size = 256;
 915                 rdev->config.cayman.sx_max_export_pos_size = 64;
 916                 rdev->config.cayman.sx_max_export_smx_size = 192;
 917                 rdev->config.cayman.max_hw_contexts = 8;
 918                 rdev->config.cayman.sq_num_cf_insts = 2;
 919
 920                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 921                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 922                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 923                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 924                 break;
 925         case CHIP_ARUBA:
 926         default:
 927                 rdev->config.cayman.max_shader_engines = 1;
 928                 rdev->config.cayman.max_pipes_per_simd = 4;
 929                 rdev->config.cayman.max_tile_pipes = 2;
 930                 if ((rdev->pdev->device == 0x9900) ||
 931                     (rdev->pdev->device == 0x9901) ||
 932                     (rdev->pdev->device == 0x9905) ||
 933                     (rdev->pdev->device == 0x9906) ||
 934                     (rdev->pdev->device == 0x9907) ||
 935                     (rdev->pdev->device == 0x9908) ||
 936                     (rdev->pdev->device == 0x9909) ||
 937                     (rdev->pdev->device == 0x990B) ||
 938                     (rdev->pdev->device == 0x990C) ||
 939                     (rdev->pdev->device == 0x990F) ||
 940                     (rdev->pdev->device == 0x9910) ||
 941                     (rdev->pdev->device == 0x9917) ||
 942                     (rdev->pdev->device == 0x9999) ||
 943                     (rdev->pdev->device == 0x999C)) {
 944                         rdev->config.cayman.max_simds_per_se = 6;
 945                         rdev->config.cayman.max_backends_per_se = 2;
 946                         rdev->config.cayman.max_hw_contexts = 8;
 947                         rdev->config.cayman.sx_max_export_size = 256;
 948                         rdev->config.cayman.sx_max_export_pos_size = 64;
 949                         rdev->config.cayman.sx_max_export_smx_size = 192;
 950                 } else if ((rdev->pdev->device == 0x9903) ||
 951                            (rdev->pdev->device == 0x9904) ||
 952                            (rdev->pdev->device == 0x990A) ||
 953                            (rdev->pdev->device == 0x990D) ||
 954                            (rdev->pdev->device == 0x990E) ||
 955                            (rdev->pdev->device == 0x9913) ||
 956                            (rdev->pdev->device == 0x9918) ||
 957                            (rdev->pdev->device == 0x999D)) {
 958                         rdev->config.cayman.max_simds_per_se = 4;
 959                         rdev->config.cayman.max_backends_per_se = 2;
 960                         rdev->config.cayman.max_hw_contexts = 8;
 961                         rdev->config.cayman.sx_max_export_size = 256;
 962                         rdev->config.cayman.sx_max_export_pos_size = 64;
 963                         rdev->config.cayman.sx_max_export_smx_size = 192;
 964                 } else if ((rdev->pdev->device == 0x9919) ||
 965                            (rdev->pdev->device == 0x9990) ||
 966                            (rdev->pdev->device == 0x9991) ||
 967                            (rdev->pdev->device == 0x9994) ||
 968                            (rdev->pdev->device == 0x9995) ||
 969                            (rdev->pdev->device == 0x9996) ||
 970                            (rdev->pdev->device == 0x999A) ||
 971                            (rdev->pdev->device == 0x99A0)) {
 972                         rdev->config.cayman.max_simds_per_se = 3;
 973                         rdev->config.cayman.max_backends_per_se = 1;
 974                         rdev->config.cayman.max_hw_contexts = 4;
 975                         rdev->config.cayman.sx_max_export_size = 128;
 976                         rdev->config.cayman.sx_max_export_pos_size = 32;
 977                         rdev->config.cayman.sx_max_export_smx_size = 96;
 978                 } else {
 979                         rdev->config.cayman.max_simds_per_se = 2;
 980                         rdev->config.cayman.max_backends_per_se = 1;
 981                         rdev->config.cayman.max_hw_contexts = 4;
 982                         rdev->config.cayman.sx_max_export_size = 128;
 983                         rdev->config.cayman.sx_max_export_pos_size = 32;
 984                         rdev->config.cayman.sx_max_export_smx_size = 96;
 985                 }
 986                 rdev->config.cayman.max_texture_channel_caches = 2;
 987                 rdev->config.cayman.max_gprs = 256;
 988                 rdev->config.cayman.max_threads = 256;
 989                 rdev->config.cayman.max_gs_threads = 32;
 990                 rdev->config.cayman.max_stack_entries = 512;
 991                 rdev->config.cayman.sx_num_of_sets = 8;
 992                 rdev->config.cayman.sq_num_cf_insts = 2;
 993
 994                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 995                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 996                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 997                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 998                 break;
 999         }
1000
1001         /* Initialize HDP */
1002         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1003                 WREG32((0x2c14 + j), 0x00000000);
1004                 WREG32((0x2c18 + j), 0x00000000);
1005                 WREG32((0x2c1c + j), 0x00000000);
1006                 WREG32((0x2c20 + j), 0x00000000);
1007                 WREG32((0x2c24 + j), 0x00000000);
1008         }
1009
1010         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1011         WREG32(SRBM_INT_CNTL, 0x1);
1012         WREG32(SRBM_INT_ACK, 0x1);
1013
1014         evergreen_fix_pci_max_read_req_size(rdev);
1015
1016         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1017         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1018
1019         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1020         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1021         if (rdev->config.cayman.mem_row_size_in_kb > 4)
1022                 rdev->config.cayman.mem_row_size_in_kb = 4;
1023         /* XXX use MC settings? */
1024         rdev->config.cayman.shader_engine_tile_size = 32;
1025         rdev->config.cayman.num_gpus = 1;
1026         rdev->config.cayman.multi_gpu_tile_size = 64;
1027
1028         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1029         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1030         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1031         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1032         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1033         rdev->config.cayman.num_shader_engines = tmp + 1;
1034         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1035         rdev->config.cayman.num_gpus = tmp + 1;
1036         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1037         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1038         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1039         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1040
1041
1042         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1043          * not have bank info, so create a custom tiling dword.
1044          * bits 3:0   num_pipes
1045          * bits 7:4   num_banks
1046          * bits 11:8  group_size
1047          * bits 15:12 row_size
1048          */
1049         rdev->config.cayman.tile_config = 0;
1050         switch (rdev->config.cayman.num_tile_pipes) {
1051         case 1:
1052         default:
1053                 rdev->config.cayman.tile_config |= (0 << 0);
1054                 break;
1055         case 2:
1056                 rdev->config.cayman.tile_config |= (1 << 0);
1057                 break;
1058         case 4:
1059                 rdev->config.cayman.tile_config |= (2 << 0);
1060                 break;
1061         case 8:
1062                 rdev->config.cayman.tile_config |= (3 << 0);
1063                 break;
1064         }
1065
1066         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1067         if (rdev->flags & RADEON_IS_IGP)
1068                 rdev->config.cayman.tile_config |= 1 << 4;
1069         else {
1070                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1071                 case 0: /* four banks */
1072                         rdev->config.cayman.tile_config |= 0 << 4;
1073                         break;
1074                 case 1: /* eight banks */
1075                         rdev->config.cayman.tile_config |= 1 << 4;
1076                         break;
1077                 case 2: /* sixteen banks */
1078                 default:
1079                         rdev->config.cayman.tile_config |= 2 << 4;
1080                         break;
1081                 }
1082         }
1083         rdev->config.cayman.tile_config |=
1084                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1085         rdev->config.cayman.tile_config |=
1086                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1087
1088         tmp = 0;
1089         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1090                 u32 rb_disable_bitmap;
1091
1092                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1093                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1094                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1095                 tmp <<= 4;
1096                 tmp |= rb_disable_bitmap;
1097         }
1098         /* enabled rb are just the one not disabled :) */
1099         disabled_rb_mask = tmp;
1100         tmp = 0;
1101         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1102                 tmp |= (1 << i);
1103         /* if all the backends are disabled, fix it up here */
1104         if ((disabled_rb_mask & tmp) == tmp) {
1105                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1106                         disabled_rb_mask &= ~(1 << i);
1107         }
1108
1109         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1110                 u32 simd_disable_bitmap;
1111
1112                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1113                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1114                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1115                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1116                 tmp <<= 16;
1117                 tmp |= simd_disable_bitmap;
1118         }
1119         rdev->config.cayman.active_simds = hweight32(~tmp);
1120
1121         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1122         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1123
1124         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1125         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1126         if (ASIC_IS_DCE6(rdev))
1127                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1128         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1129         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1130         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1131         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1132         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1133         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1134
1135         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1136             (rdev->flags & RADEON_IS_IGP)) {
1137                 if ((disabled_rb_mask & 3) == 2) {
1138                         /* RB1 disabled, RB0 enabled */
1139                         tmp = 0x00000000;
1140                 } else {
1141                         /* RB0 disabled, RB1 enabled */
1142                         tmp = 0x11111111;
1143                 }
1144         } else {
1145                 tmp = gb_addr_config & NUM_PIPES_MASK;
1146                 tmp = r6xx_remap_render_backend(rdev, tmp,
1147                                                 rdev->config.cayman.max_backends_per_se *
1148                                                 rdev->config.cayman.max_shader_engines,
1149                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1150         }
1151         rdev->config.cayman.backend_map = tmp;
1152         WREG32(GB_BACKEND_MAP, tmp);
1153
1154         cgts_tcc_disable = 0xffff0000;
1155         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1156                 cgts_tcc_disable &= ~(1 << (16 + i));
1157         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1158         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1159         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1160         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1161
1162         /* reprogram the shader complex */
1163         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1164         for (i = 0; i < 16; i++)
1165                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1166         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1167
1168         /* set HW defaults for 3D engine */
1169         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1170
1171         sx_debug_1 = RREG32(SX_DEBUG_1);
1172         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1173         WREG32(SX_DEBUG_1, sx_debug_1);
1174
1175         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1176         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1177         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1178         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1179
1180         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1181
1182         /* need to be explicitly zero-ed */
1183         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1184         WREG32(SQ_LSTMP_RING_BASE, 0);
1185         WREG32(SQ_HSTMP_RING_BASE, 0);
1186         WREG32(SQ_ESTMP_RING_BASE, 0);
1187         WREG32(SQ_GSTMP_RING_BASE, 0);
1188         WREG32(SQ_VSTMP_RING_BASE, 0);
1189         WREG32(SQ_PSTMP_RING_BASE, 0);
1190
1191         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1192
1193         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1194                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1195                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1196
1197         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1198                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1199                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1200
1201
1202         WREG32(VGT_NUM_INSTANCES, 1);
1203
1204         WREG32(CP_PERFMON_CNTL, 0);
1205
1206         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1207                                   FETCH_FIFO_HIWATER(0x4) |
1208                                   DONE_FIFO_HIWATER(0xe0) |
1209                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1210
1211         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1212         WREG32(SQ_CONFIG, (VC_ENABLE |
1213                            EXPORT_SRC_C |
1214                            GFX_PRIO(0) |
1215                            CS1_PRIO(0) |
1216                            CS2_PRIO(1)));
1217         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1218
1219         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1220                                           FORCE_EOV_MAX_REZ_CNT(255)));
1221
1222         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1223                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1224
1225         WREG32(VGT_GS_VERTEX_REUSE, 16);
1226         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1227
1228         WREG32(CB_PERF_CTR0_SEL_0, 0);
1229         WREG32(CB_PERF_CTR0_SEL_1, 0);
1230         WREG32(CB_PERF_CTR1_SEL_0, 0);
1231         WREG32(CB_PERF_CTR1_SEL_1, 0);
1232         WREG32(CB_PERF_CTR2_SEL_0, 0);
1233         WREG32(CB_PERF_CTR2_SEL_1, 0);
1234         WREG32(CB_PERF_CTR3_SEL_0, 0);
1235         WREG32(CB_PERF_CTR3_SEL_1, 0);
1236
1237         tmp = RREG32(HDP_MISC_CNTL);
1238         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1239         WREG32(HDP_MISC_CNTL, tmp);
1240
1241         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1242         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1243
1244         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1245
1246         udelay(50);
1247
1248         /* set clockgating golden values on TN */
1249         if (rdev->family == CHIP_ARUBA) {
1250                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1251                 tmp &= ~0x00380000;
1252                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1253                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1254                 tmp &= ~0x0e000000;
1255                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1256         }
1257 }
1258
1259 /*
1260  * GART
1261  */
1262 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1263 {
1264         /* flush hdp cache */
1265         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1266
1267         /* bits 0-7 are the VM contexts0-7 */
1268         WREG32(VM_INVALIDATE_REQUEST, 1);
1269 }
1270
1271 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1272 {
1273         int i, r;
1274
1275         if (rdev->gart.robj == NULL) {
1276                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1277                 return -EINVAL;
1278         }
1279         r = radeon_gart_table_vram_pin(rdev);
1280         if (r)
1281                 return r;
1282         /* Setup TLB control */
1283         WREG32(MC_VM_MX_L1_TLB_CNTL,
1284                (0xA << 7) |
1285                ENABLE_L1_TLB |
1286                ENABLE_L1_FRAGMENT_PROCESSING |
1287                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1288                ENABLE_ADVANCED_DRIVER_MODEL |
1289                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1290         /* Setup L2 cache */
1291         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1292                ENABLE_L2_FRAGMENT_PROCESSING |
1293                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1294                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1295                EFFECTIVE_L2_QUEUE_SIZE(7) |
1296                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1297         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1298         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1299                BANK_SELECT(6) |
1300                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1301         /* setup context0 */
1302         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1303         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1304         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1305         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1306                         (u32)(rdev->dummy_page.addr >> 12));
1307         WREG32(VM_CONTEXT0_CNTL2, 0);
1308         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1309                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1310
1311         WREG32(0x15D4, 0);
1312         WREG32(0x15D8, 0);
1313         WREG32(0x15DC, 0);
1314
1315         /* empty context1-7 */
1316         /* Assign the pt base to something valid for now; the pts used for
1317          * the VMs are determined by the application and setup and assigned
1318          * on the fly in the vm part of radeon_gart.c
1319          */
1320         for (i = 1; i < 8; i++) {
1321                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1322                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1323                         rdev->vm_manager.max_pfn - 1);
1324                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1325                        rdev->vm_manager.saved_table_addr[i]);
1326         }
1327
1328         /* enable context1-7 */
1329         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1330                (u32)(rdev->dummy_page.addr >> 12));
1331         WREG32(VM_CONTEXT1_CNTL2, 4);
1332         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1333                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1334                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1335                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1336                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1337                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1338                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1339                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1340                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1341                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1342                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1343                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1344                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1345                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1346
1347         cayman_pcie_gart_tlb_flush(rdev);
1348         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1349                  (unsigned)(rdev->mc.gtt_size >> 20),
1350                  (unsigned long long)rdev->gart.table_addr);
1351         rdev->gart.ready = true;
1352         return 0;
1353 }
1354
1355 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1356 {
1357         unsigned i;
1358
1359         for (i = 1; i < 8; ++i) {
1360                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1361                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1362         }
1363
1364         /* Disable all tables */
1365         WREG32(VM_CONTEXT0_CNTL, 0);
1366         WREG32(VM_CONTEXT1_CNTL, 0);
1367         /* Setup TLB control */
1368         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1369                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1370                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1371         /* Setup L2 cache */
1372         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1373                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1374                EFFECTIVE_L2_QUEUE_SIZE(7) |
1375                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1376         WREG32(VM_L2_CNTL2, 0);
1377         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1378                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1379         radeon_gart_table_vram_unpin(rdev);
1380 }
1381
1382 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1383 {
1384         cayman_pcie_gart_disable(rdev);
1385         radeon_gart_table_vram_free(rdev);
1386         radeon_gart_fini(rdev);
1387 }
1388
1389 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1390                               int ring, u32 cp_int_cntl)
1391 {
1392         WREG32(SRBM_GFX_CNTL, RINGID(ring));
1393         WREG32(CP_INT_CNTL, cp_int_cntl);
1394 }
1395
1396 /*
1397  * CP.
1398  */
1399 void cayman_fence_ring_emit(struct radeon_device *rdev,
1400                             struct radeon_fence *fence)
1401 {
1402         struct radeon_ring *ring = &rdev->ring[fence->ring];
1403         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1404         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1405                 PACKET3_SH_ACTION_ENA;
1406
1407         /* flush read cache over gart for this vmid */
1408         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1409         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1410         radeon_ring_write(ring, 0xFFFFFFFF);
1411         radeon_ring_write(ring, 0);
1412         radeon_ring_write(ring, 10); /* poll interval */
1413         /* EVENT_WRITE_EOP - flush caches, send int */
1414         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1415         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1416         radeon_ring_write(ring, lower_32_bits(addr));
1417         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1418         radeon_ring_write(ring, fence->seq);
1419         radeon_ring_write(ring, 0);
1420 }
1421
1422 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1423 {
1424         struct radeon_ring *ring = &rdev->ring[ib->ring];
1425         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1426         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1427                 PACKET3_SH_ACTION_ENA;
1428
1429         /* set to DX10/11 mode */
1430         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1431         radeon_ring_write(ring, 1);
1432
1433         if (ring->rptr_save_reg) {
1434                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1435                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1436                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1437                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1438                 radeon_ring_write(ring, next_rptr);
1439         }
1440
1441         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1442         radeon_ring_write(ring,
1443 #ifdef __BIG_ENDIAN
1444                           (2 << 0) |
1445 #endif
1446                           (ib->gpu_addr & 0xFFFFFFFC));
1447         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1448         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1449
1450         /* flush read cache over gart for this vmid */
1451         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1452         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1453         radeon_ring_write(ring, 0xFFFFFFFF);
1454         radeon_ring_write(ring, 0);
1455         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1456 }
1457
1458 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1459 {
1460         if (enable)
1461                 WREG32(CP_ME_CNTL, 0);
1462         else {
1463                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1464                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1465                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1466                 WREG32(SCRATCH_UMSK, 0);
1467                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1468         }
1469 }
1470
1471 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1472                         struct radeon_ring *ring)
1473 {
1474         u32 rptr;
1475
1476         if (rdev->wb.enabled)
1477                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1478         else {
1479                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1480                         rptr = RREG32(CP_RB0_RPTR);
1481                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1482                         rptr = RREG32(CP_RB1_RPTR);
1483                 else
1484                         rptr = RREG32(CP_RB2_RPTR);
1485         }
1486
1487         return rptr;
1488 }
1489
1490 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1491                         struct radeon_ring *ring)
1492 {
1493         u32 wptr;
1494
1495         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1496                 wptr = RREG32(CP_RB0_WPTR);
1497         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1498                 wptr = RREG32(CP_RB1_WPTR);
1499         else
1500                 wptr = RREG32(CP_RB2_WPTR);
1501
1502         return wptr;
1503 }
1504
1505 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1506                          struct radeon_ring *ring)
1507 {
1508         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1509                 WREG32(CP_RB0_WPTR, ring->wptr);
1510                 (void)RREG32(CP_RB0_WPTR);
1511         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1512                 WREG32(CP_RB1_WPTR, ring->wptr);
1513                 (void)RREG32(CP_RB1_WPTR);
1514         } else {
1515                 WREG32(CP_RB2_WPTR, ring->wptr);
1516                 (void)RREG32(CP_RB2_WPTR);
1517         }
1518 }
1519
1520 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1521 {
1522         const __be32 *fw_data;
1523         int i;
1524
1525         if (!rdev->me_fw || !rdev->pfp_fw)
1526                 return -EINVAL;
1527
1528         cayman_cp_enable(rdev, false);
1529
1530         fw_data = (const __be32 *)rdev->pfp_fw->data;
1531         WREG32(CP_PFP_UCODE_ADDR, 0);
1532         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1533                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1534         WREG32(CP_PFP_UCODE_ADDR, 0);
1535
1536         fw_data = (const __be32 *)rdev->me_fw->data;
1537         WREG32(CP_ME_RAM_WADDR, 0);
1538         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1539                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1540
1541         WREG32(CP_PFP_UCODE_ADDR, 0);
1542         WREG32(CP_ME_RAM_WADDR, 0);
1543         WREG32(CP_ME_RAM_RADDR, 0);
1544         return 0;
1545 }
1546
1547 static int cayman_cp_start(struct radeon_device *rdev)
1548 {
1549         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1550         int r, i;
1551
1552         r = radeon_ring_lock(rdev, ring, 7);
1553         if (r) {
1554                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1555                 return r;
1556         }
1557         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1558         radeon_ring_write(ring, 0x1);
1559         radeon_ring_write(ring, 0x0);
1560         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1561         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1562         radeon_ring_write(ring, 0);
1563         radeon_ring_write(ring, 0);
1564         radeon_ring_unlock_commit(rdev, ring, false);
1565
1566         cayman_cp_enable(rdev, true);
1567
1568         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1569         if (r) {
1570                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1571                 return r;
1572         }
1573
1574         /* setup clear context state */
1575         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1576         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1577
1578         for (i = 0; i < cayman_default_size; i++)
1579                 radeon_ring_write(ring, cayman_default_state[i]);
1580
1581         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1582         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1583
1584         /* set clear context state */
1585         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1586         radeon_ring_write(ring, 0);
1587
1588         /* SQ_VTX_BASE_VTX_LOC */
1589         radeon_ring_write(ring, 0xc0026f00);
1590         radeon_ring_write(ring, 0x00000000);
1591         radeon_ring_write(ring, 0x00000000);
1592         radeon_ring_write(ring, 0x00000000);
1593
1594         /* Clear consts */
1595         radeon_ring_write(ring, 0xc0036f00);
1596         radeon_ring_write(ring, 0x00000bc4);
1597         radeon_ring_write(ring, 0xffffffff);
1598         radeon_ring_write(ring, 0xffffffff);
1599         radeon_ring_write(ring, 0xffffffff);
1600
1601         radeon_ring_write(ring, 0xc0026900);
1602         radeon_ring_write(ring, 0x00000316);
1603         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1604         radeon_ring_write(ring, 0x00000010); /*  */
1605
1606         radeon_ring_unlock_commit(rdev, ring, false);
1607
1608         /* XXX init other rings */
1609
1610         return 0;
1611 }
1612
1613 static void cayman_cp_fini(struct radeon_device *rdev)
1614 {
1615         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1616         cayman_cp_enable(rdev, false);
1617         radeon_ring_fini(rdev, ring);
1618         radeon_scratch_free(rdev, ring->rptr_save_reg);
1619 }
1620
1621 static int cayman_cp_resume(struct radeon_device *rdev)
1622 {
1623         static const int ridx[] = {
1624                 RADEON_RING_TYPE_GFX_INDEX,
1625                 CAYMAN_RING_TYPE_CP1_INDEX,
1626                 CAYMAN_RING_TYPE_CP2_INDEX
1627         };
1628         static const unsigned cp_rb_cntl[] = {
1629                 CP_RB0_CNTL,
1630                 CP_RB1_CNTL,
1631                 CP_RB2_CNTL,
1632         };
1633         static const unsigned cp_rb_rptr_addr[] = {
1634                 CP_RB0_RPTR_ADDR,
1635                 CP_RB1_RPTR_ADDR,
1636                 CP_RB2_RPTR_ADDR
1637         };
1638         static const unsigned cp_rb_rptr_addr_hi[] = {
1639                 CP_RB0_RPTR_ADDR_HI,
1640                 CP_RB1_RPTR_ADDR_HI,
1641                 CP_RB2_RPTR_ADDR_HI
1642         };
1643         static const unsigned cp_rb_base[] = {
1644                 CP_RB0_BASE,
1645                 CP_RB1_BASE,
1646                 CP_RB2_BASE
1647         };
1648         static const unsigned cp_rb_rptr[] = {
1649                 CP_RB0_RPTR,
1650                 CP_RB1_RPTR,
1651                 CP_RB2_RPTR
1652         };
1653         static const unsigned cp_rb_wptr[] = {
1654                 CP_RB0_WPTR,
1655                 CP_RB1_WPTR,
1656                 CP_RB2_WPTR
1657         };
1658         struct radeon_ring *ring;
1659         int i, r;
1660
1661         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1662         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1663                                  SOFT_RESET_PA |
1664                                  SOFT_RESET_SH |
1665                                  SOFT_RESET_VGT |
1666                                  SOFT_RESET_SPI |
1667                                  SOFT_RESET_SX));
1668         RREG32(GRBM_SOFT_RESET);
1669         mdelay(15);
1670         WREG32(GRBM_SOFT_RESET, 0);
1671         RREG32(GRBM_SOFT_RESET);
1672
1673         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1674         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1675
1676         /* Set the write pointer delay */
1677         WREG32(CP_RB_WPTR_DELAY, 0);
1678
1679         WREG32(CP_DEBUG, (1 << 27));
1680
1681         /* set the wb address whether it's enabled or not */
1682         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1683         WREG32(SCRATCH_UMSK, 0xff);
1684
1685         for (i = 0; i < 3; ++i) {
1686                 uint32_t rb_cntl;
1687                 uint64_t addr;
1688
1689                 /* Set ring buffer size */
1690                 ring = &rdev->ring[ridx[i]];
1691                 rb_cntl = order_base_2(ring->ring_size / 8);
1692                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1693 #ifdef __BIG_ENDIAN
1694                 rb_cntl |= BUF_SWAP_32BIT;
1695 #endif
1696                 WREG32(cp_rb_cntl[i], rb_cntl);
1697
1698                 /* set the wb address whether it's enabled or not */
1699                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1700                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1701                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1702         }
1703
1704         /* set the rb base addr, this causes an internal reset of ALL rings */
1705         for (i = 0; i < 3; ++i) {
1706                 ring = &rdev->ring[ridx[i]];
1707                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1708         }
1709
1710         for (i = 0; i < 3; ++i) {
1711                 /* Initialize the ring buffer's read and write pointers */
1712                 ring = &rdev->ring[ridx[i]];
1713                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1714
1715                 ring->wptr = 0;
1716                 WREG32(cp_rb_rptr[i], 0);
1717                 WREG32(cp_rb_wptr[i], ring->wptr);
1718
1719                 mdelay(1);
1720                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1721         }
1722
1723         /* start the rings */
1724         cayman_cp_start(rdev);
1725         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1726         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1727         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1728         /* this only test cp0 */
1729         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1730         if (r) {
1731                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1732                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1733                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1734                 return r;
1735         }
1736
1737         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1738                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1739
1740         return 0;
1741 }
1742
1743 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1744 {
1745         u32 reset_mask = 0;
1746         u32 tmp;
1747
1748         /* GRBM_STATUS */
1749         tmp = RREG32(GRBM_STATUS);
1750         if (tmp & (PA_BUSY | SC_BUSY |
1751                    SH_BUSY | SX_BUSY |
1752                    TA_BUSY | VGT_BUSY |
1753                    DB_BUSY | CB_BUSY |
1754                    GDS_BUSY | SPI_BUSY |
1755                    IA_BUSY | IA_BUSY_NO_DMA))
1756                 reset_mask |= RADEON_RESET_GFX;
1757
1758         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1759                    CP_BUSY | CP_COHERENCY_BUSY))
1760                 reset_mask |= RADEON_RESET_CP;
1761
1762         if (tmp & GRBM_EE_BUSY)
1763                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1764
1765         /* DMA_STATUS_REG 0 */
1766         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1767         if (!(tmp & DMA_IDLE))
1768                 reset_mask |= RADEON_RESET_DMA;
1769
1770         /* DMA_STATUS_REG 1 */
1771         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1772         if (!(tmp & DMA_IDLE))
1773                 reset_mask |= RADEON_RESET_DMA1;
1774
1775         /* SRBM_STATUS2 */
1776         tmp = RREG32(SRBM_STATUS2);
1777         if (tmp & DMA_BUSY)
1778                 reset_mask |= RADEON_RESET_DMA;
1779
1780         if (tmp & DMA1_BUSY)
1781                 reset_mask |= RADEON_RESET_DMA1;
1782
1783         /* SRBM_STATUS */
1784         tmp = RREG32(SRBM_STATUS);
1785         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1786                 reset_mask |= RADEON_RESET_RLC;
1787
1788         if (tmp & IH_BUSY)
1789                 reset_mask |= RADEON_RESET_IH;
1790
1791         if (tmp & SEM_BUSY)
1792                 reset_mask |= RADEON_RESET_SEM;
1793
1794         if (tmp & GRBM_RQ_PENDING)
1795                 reset_mask |= RADEON_RESET_GRBM;
1796
1797         if (tmp & VMC_BUSY)
1798                 reset_mask |= RADEON_RESET_VMC;
1799
1800         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1801                    MCC_BUSY | MCD_BUSY))
1802                 reset_mask |= RADEON_RESET_MC;
1803
1804         if (evergreen_is_display_hung(rdev))
1805                 reset_mask |= RADEON_RESET_DISPLAY;
1806
1807         /* VM_L2_STATUS */
1808         tmp = RREG32(VM_L2_STATUS);
1809         if (tmp & L2_BUSY)
1810                 reset_mask |= RADEON_RESET_VMC;
1811
1812         /* Skip MC reset as it's mostly likely not hung, just busy */
1813         if (reset_mask & RADEON_RESET_MC) {
1814                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1815                 reset_mask &= ~RADEON_RESET_MC;
1816         }
1817
1818         return reset_mask;
1819 }
1820
1821 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1822 {
1823         struct evergreen_mc_save save;
1824         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1825         u32 tmp;
1826
1827         if (reset_mask == 0)
1828                 return;
1829
1830         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1831
1832         evergreen_print_gpu_status_regs(rdev);
1833         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1834                  RREG32(0x14F8));
1835         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1836                  RREG32(0x14D8));
1837         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1838                  RREG32(0x14FC));
1839         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1840                  RREG32(0x14DC));
1841
1842         /* Disable CP parsing/prefetching */
1843         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1844
1845         if (reset_mask & RADEON_RESET_DMA) {
1846                 /* dma0 */
1847                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1848                 tmp &= ~DMA_RB_ENABLE;
1849                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1850         }
1851
1852         if (reset_mask & RADEON_RESET_DMA1) {
1853                 /* dma1 */
1854                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1855                 tmp &= ~DMA_RB_ENABLE;
1856                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1857         }
1858
1859         udelay(50);
1860
1861         evergreen_mc_stop(rdev, &save);
1862         if (evergreen_mc_wait_for_idle(rdev)) {
1863                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1864         }
1865
1866         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1867                 grbm_soft_reset = SOFT_RESET_CB |
1868                         SOFT_RESET_DB |
1869                         SOFT_RESET_GDS |
1870                         SOFT_RESET_PA |
1871                         SOFT_RESET_SC |
1872                         SOFT_RESET_SPI |
1873                         SOFT_RESET_SH |
1874                         SOFT_RESET_SX |
1875                         SOFT_RESET_TC |
1876                         SOFT_RESET_TA |
1877                         SOFT_RESET_VGT |
1878                         SOFT_RESET_IA;
1879         }
1880
1881         if (reset_mask & RADEON_RESET_CP) {
1882                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1883
1884                 srbm_soft_reset |= SOFT_RESET_GRBM;
1885         }
1886
1887         if (reset_mask & RADEON_RESET_DMA)
1888                 srbm_soft_reset |= SOFT_RESET_DMA;
1889
1890         if (reset_mask & RADEON_RESET_DMA1)
1891                 srbm_soft_reset |= SOFT_RESET_DMA1;
1892
1893         if (reset_mask & RADEON_RESET_DISPLAY)
1894                 srbm_soft_reset |= SOFT_RESET_DC;
1895
1896         if (reset_mask & RADEON_RESET_RLC)
1897                 srbm_soft_reset |= SOFT_RESET_RLC;
1898
1899         if (reset_mask & RADEON_RESET_SEM)
1900                 srbm_soft_reset |= SOFT_RESET_SEM;
1901
1902         if (reset_mask & RADEON_RESET_IH)
1903                 srbm_soft_reset |= SOFT_RESET_IH;
1904
1905         if (reset_mask & RADEON_RESET_GRBM)
1906                 srbm_soft_reset |= SOFT_RESET_GRBM;
1907
1908         if (reset_mask & RADEON_RESET_VMC)
1909                 srbm_soft_reset |= SOFT_RESET_VMC;
1910
1911         if (!(rdev->flags & RADEON_IS_IGP)) {
1912                 if (reset_mask & RADEON_RESET_MC)
1913                         srbm_soft_reset |= SOFT_RESET_MC;
1914         }
1915
1916         if (grbm_soft_reset) {
1917                 tmp = RREG32(GRBM_SOFT_RESET);
1918                 tmp |= grbm_soft_reset;
1919                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1920                 WREG32(GRBM_SOFT_RESET, tmp);
1921                 tmp = RREG32(GRBM_SOFT_RESET);
1922
1923                 udelay(50);
1924
1925                 tmp &= ~grbm_soft_reset;
1926                 WREG32(GRBM_SOFT_RESET, tmp);
1927                 tmp = RREG32(GRBM_SOFT_RESET);
1928         }
1929
1930         if (srbm_soft_reset) {
1931                 tmp = RREG32(SRBM_SOFT_RESET);
1932                 tmp |= srbm_soft_reset;
1933                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1934                 WREG32(SRBM_SOFT_RESET, tmp);
1935                 tmp = RREG32(SRBM_SOFT_RESET);
1936
1937                 udelay(50);
1938
1939                 tmp &= ~srbm_soft_reset;
1940                 WREG32(SRBM_SOFT_RESET, tmp);
1941                 tmp = RREG32(SRBM_SOFT_RESET);
1942         }
1943
1944         /* Wait a little for things to settle down */
1945         udelay(50);
1946
1947         evergreen_mc_resume(rdev, &save);
1948         udelay(50);
1949
1950         evergreen_print_gpu_status_regs(rdev);
1951 }
1952
1953 int cayman_asic_reset(struct radeon_device *rdev, bool hard)
1954 {
1955         u32 reset_mask;
1956
1957         if (hard) {
1958                 evergreen_gpu_pci_config_reset(rdev);
1959                 return 0;
1960         }
1961
1962         reset_mask = cayman_gpu_check_soft_reset(rdev);
1963
1964         if (reset_mask)
1965                 r600_set_bios_scratch_engine_hung(rdev, true);
1966
1967         cayman_gpu_soft_reset(rdev, reset_mask);
1968
1969         reset_mask = cayman_gpu_check_soft_reset(rdev);
1970
1971         if (reset_mask)
1972                 evergreen_gpu_pci_config_reset(rdev);
1973
1974         r600_set_bios_scratch_engine_hung(rdev, false);
1975
1976         return 0;
1977 }
1978
1979 /**
1980  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1981  *
1982  * @rdev: radeon_device pointer
1983  * @ring: radeon_ring structure holding ring information
1984  *
1985  * Check if the GFX engine is locked up.
1986  * Returns true if the engine appears to be locked up, false if not.
1987  */
1988 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1989 {
1990         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1991
1992         if (!(reset_mask & (RADEON_RESET_GFX |
1993                             RADEON_RESET_COMPUTE |
1994                             RADEON_RESET_CP))) {
1995                 radeon_ring_lockup_update(rdev, ring);
1996                 return false;
1997         }
1998         return radeon_ring_test_lockup(rdev, ring);
1999 }
2000
2001 static void cayman_uvd_init(struct radeon_device *rdev)
2002 {
2003         int r;
2004
2005         if (!rdev->has_uvd)
2006                 return;
2007
2008         r = radeon_uvd_init(rdev);
2009         if (r) {
2010                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
2011                 /*
2012                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
2013                  * to early fails uvd_v2_2_resume() and thus nothing happens
2014                  * there. So it is pointless to try to go through that code
2015                  * hence why we disable uvd here.
2016                  */
2017                 rdev->has_uvd = 0;
2018                 return;
2019         }
2020         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
2021         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
2022 }
2023
2024 static void cayman_uvd_start(struct radeon_device *rdev)
2025 {
2026         int r;
2027
2028         if (!rdev->has_uvd)
2029                 return;
2030
2031         r = uvd_v2_2_resume(rdev);
2032         if (r) {
2033                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
2034                 goto error;
2035         }
2036         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
2037         if (r) {
2038                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
2039                 goto error;
2040         }
2041         return;
2042
2043 error:
2044         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2045 }
2046
2047 static void cayman_uvd_resume(struct radeon_device *rdev)
2048 {
2049         struct radeon_ring *ring;
2050         int r;
2051
2052         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
2053                 return;
2054
2055         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2056         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
2057         if (r) {
2058                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
2059                 return;
2060         }
2061         r = uvd_v1_0_init(rdev);
2062         if (r) {
2063                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
2064                 return;
2065         }
2066 }
2067
2068 static void cayman_vce_init(struct radeon_device *rdev)
2069 {
2070         int r;
2071
2072         /* Only set for CHIP_ARUBA */
2073         if (!rdev->has_vce)
2074                 return;
2075
2076         r = radeon_vce_init(rdev);
2077         if (r) {
2078                 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
2079                 /*
2080                  * At this point rdev->vce.vcpu_bo is NULL which trickles down
2081                  * to early fails cayman_vce_start() and thus nothing happens
2082                  * there. So it is pointless to try to go through that code
2083                  * hence why we disable vce here.
2084                  */
2085                 rdev->has_vce = 0;
2086                 return;
2087         }
2088         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
2089         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
2090         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
2091         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
2092 }
2093
2094 static void cayman_vce_start(struct radeon_device *rdev)
2095 {
2096         int r;
2097
2098         if (!rdev->has_vce)
2099                 return;
2100
2101         r = radeon_vce_resume(rdev);
2102         if (r) {
2103                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2104                 goto error;
2105         }
2106         r = vce_v1_0_resume(rdev);
2107         if (r) {
2108                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2109                 goto error;
2110         }
2111         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
2112         if (r) {
2113                 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
2114                 goto error;
2115         }
2116         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
2117         if (r) {
2118                 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
2119                 goto error;
2120         }
2121         return;
2122
2123 error:
2124         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2125         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2126 }
2127
2128 static void cayman_vce_resume(struct radeon_device *rdev)
2129 {
2130         struct radeon_ring *ring;
2131         int r;
2132
2133         if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
2134                 return;
2135
2136         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2137         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2138         if (r) {
2139                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2140                 return;
2141         }
2142         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2143         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2144         if (r) {
2145                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2146                 return;
2147         }
2148         r = vce_v1_0_init(rdev);
2149         if (r) {
2150                 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
2151                 return;
2152         }
2153 }
2154
2155 static int cayman_startup(struct radeon_device *rdev)
2156 {
2157         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2158         int r;
2159
2160         /* enable pcie gen2 link */
2161         evergreen_pcie_gen2_enable(rdev);
2162         /* enable aspm */
2163         evergreen_program_aspm(rdev);
2164
2165         /* scratch needs to be initialized before MC */
2166         r = r600_vram_scratch_init(rdev);
2167         if (r)
2168                 return r;
2169
2170         evergreen_mc_program(rdev);
2171
2172         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2173                 r = ni_mc_load_microcode(rdev);
2174                 if (r) {
2175                         DRM_ERROR("Failed to load MC firmware!\n");
2176                         return r;
2177                 }
2178         }
2179
2180         r = cayman_pcie_gart_enable(rdev);
2181         if (r)
2182                 return r;
2183         cayman_gpu_init(rdev);
2184
2185         /* allocate rlc buffers */
2186         if (rdev->flags & RADEON_IS_IGP) {
2187                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2188                 rdev->rlc.reg_list_size =
2189                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2190                 rdev->rlc.cs_data = cayman_cs_data;
2191                 r = sumo_rlc_init(rdev);
2192                 if (r) {
2193                         DRM_ERROR("Failed to init rlc BOs!\n");
2194                         return r;
2195                 }
2196         }
2197
2198         /* allocate wb buffer */
2199         r = radeon_wb_init(rdev);
2200         if (r)
2201                 return r;
2202
2203         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2204         if (r) {
2205                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2206                 return r;
2207         }
2208
2209         cayman_uvd_start(rdev);
2210         cayman_vce_start(rdev);
2211
2212         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2213         if (r) {
2214                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2215                 return r;
2216         }
2217
2218         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2219         if (r) {
2220                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2221                 return r;
2222         }
2223
2224         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2225         if (r) {
2226                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2227                 return r;
2228         }
2229
2230         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2231         if (r) {
2232                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2233                 return r;
2234         }
2235
2236         /* Enable IRQ */
2237         if (!rdev->irq.installed) {
2238                 r = radeon_irq_kms_init(rdev);
2239                 if (r)
2240                         return r;
2241         }
2242
2243         r = r600_irq_init(rdev);
2244         if (r) {
2245                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2246                 radeon_irq_kms_fini(rdev);
2247                 return r;
2248         }
2249         evergreen_irq_set(rdev);
2250
2251         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2252                              RADEON_CP_PACKET2);
2253         if (r)
2254                 return r;
2255
2256         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2257         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2258                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2259         if (r)
2260                 return r;
2261
2262         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2263         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2264                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2265         if (r)
2266                 return r;
2267
2268         r = cayman_cp_load_microcode(rdev);
2269         if (r)
2270                 return r;
2271         r = cayman_cp_resume(rdev);
2272         if (r)
2273                 return r;
2274
2275         r = cayman_dma_resume(rdev);
2276         if (r)
2277                 return r;
2278
2279         cayman_uvd_resume(rdev);
2280         cayman_vce_resume(rdev);
2281
2282         r = radeon_ib_pool_init(rdev);
2283         if (r) {
2284                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2285                 return r;
2286         }
2287
2288         r = radeon_vm_manager_init(rdev);
2289         if (r) {
2290                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2291                 return r;
2292         }
2293
2294         r = radeon_audio_init(rdev);
2295         if (r)
2296                 return r;
2297
2298         return 0;
2299 }
2300
2301 int cayman_resume(struct radeon_device *rdev)
2302 {
2303         int r;
2304
2305         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2306          * posting will perform necessary task to bring back GPU into good
2307          * shape.
2308          */
2309         /* post card */
2310         atom_asic_init(rdev->mode_info.atom_context);
2311
2312         /* init golden registers */
2313         ni_init_golden_registers(rdev);
2314
2315         if (rdev->pm.pm_method == PM_METHOD_DPM)
2316                 radeon_pm_resume(rdev);
2317
2318         rdev->accel_working = true;
2319         r = cayman_startup(rdev);
2320         if (r) {
2321                 DRM_ERROR("cayman startup failed on resume\n");
2322                 rdev->accel_working = false;
2323                 return r;
2324         }
2325         return r;
2326 }
2327
2328 int cayman_suspend(struct radeon_device *rdev)
2329 {
2330         radeon_pm_suspend(rdev);
2331         radeon_audio_fini(rdev);
2332         radeon_vm_manager_fini(rdev);
2333         cayman_cp_enable(rdev, false);
2334         cayman_dma_stop(rdev);
2335         if (rdev->has_uvd) {
2336                 uvd_v1_0_fini(rdev);
2337                 radeon_uvd_suspend(rdev);
2338         }
2339         evergreen_irq_suspend(rdev);
2340         radeon_wb_disable(rdev);
2341         cayman_pcie_gart_disable(rdev);
2342         return 0;
2343 }
2344
2345 /* Plan is to move initialization in that function and use
2346  * helper function so that radeon_device_init pretty much
2347  * do nothing more than calling asic specific function. This
2348  * should also allow to remove a bunch of callback function
2349  * like vram_info.
2350  */
2351 int cayman_init(struct radeon_device *rdev)
2352 {
2353         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2354         int r;
2355
2356         /* Read BIOS */
2357         if (!radeon_get_bios(rdev)) {
2358                 if (ASIC_IS_AVIVO(rdev))
2359                         return -EINVAL;
2360         }
2361         /* Must be an ATOMBIOS */
2362         if (!rdev->is_atom_bios) {
2363                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2364                 return -EINVAL;
2365         }
2366         r = radeon_atombios_init(rdev);
2367         if (r)
2368                 return r;
2369
2370         /* Post card if necessary */
2371         if (!radeon_card_posted(rdev)) {
2372                 if (!rdev->bios) {
2373                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2374                         return -EINVAL;
2375                 }
2376                 DRM_INFO("GPU not posted. posting now...\n");
2377                 atom_asic_init(rdev->mode_info.atom_context);
2378         }
2379         /* init golden registers */
2380         ni_init_golden_registers(rdev);
2381         /* Initialize scratch registers */
2382         r600_scratch_init(rdev);
2383         /* Initialize surface registers */
2384         radeon_surface_init(rdev);
2385         /* Initialize clocks */
2386         radeon_get_clock_info(rdev->ddev);
2387         /* Fence driver */
2388         r = radeon_fence_driver_init(rdev);
2389         if (r)
2390                 return r;
2391         /* initialize memory controller */
2392         r = evergreen_mc_init(rdev);
2393         if (r)
2394                 return r;
2395         /* Memory manager */
2396         r = radeon_bo_init(rdev);
2397         if (r)
2398                 return r;
2399
2400         if (rdev->flags & RADEON_IS_IGP) {
2401                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2402                         r = ni_init_microcode(rdev);
2403                         if (r) {
2404                                 DRM_ERROR("Failed to load firmware!\n");
2405                                 return r;
2406                         }
2407                 }
2408         } else {
2409                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2410                         r = ni_init_microcode(rdev);
2411                         if (r) {
2412                                 DRM_ERROR("Failed to load firmware!\n");
2413                                 return r;
2414                         }
2415                 }
2416         }
2417
2418         /* Initialize power management */
2419         radeon_pm_init(rdev);
2420
2421         ring->ring_obj = NULL;
2422         r600_ring_init(rdev, ring, 1024 * 1024);
2423
2424         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2425         ring->ring_obj = NULL;
2426         r600_ring_init(rdev, ring, 64 * 1024);
2427
2428         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2429         ring->ring_obj = NULL;
2430         r600_ring_init(rdev, ring, 64 * 1024);
2431
2432         cayman_uvd_init(rdev);
2433         cayman_vce_init(rdev);
2434
2435         rdev->ih.ring_obj = NULL;
2436         r600_ih_ring_init(rdev, 64 * 1024);
2437
2438         r = r600_pcie_gart_init(rdev);
2439         if (r)
2440                 return r;
2441
2442         rdev->accel_working = true;
2443         r = cayman_startup(rdev);
2444         if (r) {
2445                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2446                 cayman_cp_fini(rdev);
2447                 cayman_dma_fini(rdev);
2448                 r600_irq_fini(rdev);
2449                 if (rdev->flags & RADEON_IS_IGP)
2450                         sumo_rlc_fini(rdev);
2451                 radeon_wb_fini(rdev);
2452                 radeon_ib_pool_fini(rdev);
2453                 radeon_vm_manager_fini(rdev);
2454                 radeon_irq_kms_fini(rdev);
2455                 cayman_pcie_gart_fini(rdev);
2456                 rdev->accel_working = false;
2457         }
2458
2459         /* Don't start up if the MC ucode is missing.
2460          * The default clocks and voltages before the MC ucode
2461          * is loaded are not suffient for advanced operations.
2462          *
2463          * We can skip this check for TN, because there is no MC
2464          * ucode.
2465          */
2466         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2467                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2468                 return -EINVAL;
2469         }
2470
2471         return 0;
2472 }
2473
2474 void cayman_fini(struct radeon_device *rdev)
2475 {
2476         radeon_pm_fini(rdev);
2477         cayman_cp_fini(rdev);
2478         cayman_dma_fini(rdev);
2479         r600_irq_fini(rdev);
2480         if (rdev->flags & RADEON_IS_IGP)
2481                 sumo_rlc_fini(rdev);
2482         radeon_wb_fini(rdev);
2483         radeon_vm_manager_fini(rdev);
2484         radeon_ib_pool_fini(rdev);
2485         radeon_irq_kms_fini(rdev);
2486         uvd_v1_0_fini(rdev);
2487         radeon_uvd_fini(rdev);
2488         if (rdev->has_vce)
2489                 radeon_vce_fini(rdev);
2490         cayman_pcie_gart_fini(rdev);
2491         r600_vram_scratch_fini(rdev);
2492         radeon_gem_fini(rdev);
2493         radeon_fence_driver_fini(rdev);
2494         radeon_bo_fini(rdev);
2495         radeon_atombios_fini(rdev);
2496         kfree(rdev->bios);
2497         rdev->bios = NULL;
2498 }
2499
2500 /*
2501  * vm
2502  */
2503 int cayman_vm_init(struct radeon_device *rdev)
2504 {
2505         /* number of VMs */
2506         rdev->vm_manager.nvm = 8;
2507         /* base offset of vram pages */
2508         if (rdev->flags & RADEON_IS_IGP) {
2509                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2510                 tmp <<= 22;
2511                 rdev->vm_manager.vram_base_offset = tmp;
2512         } else
2513                 rdev->vm_manager.vram_base_offset = 0;
2514         return 0;
2515 }
2516
2517 void cayman_vm_fini(struct radeon_device *rdev)
2518 {
2519 }
2520
2521 /**
2522  * cayman_vm_decode_fault - print human readable fault info
2523  *
2524  * @rdev: radeon_device pointer
2525  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2526  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2527  *
2528  * Print human readable fault information (cayman/TN).
2529  */
2530 void cayman_vm_decode_fault(struct radeon_device *rdev,
2531                             u32 status, u32 addr)
2532 {
2533         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2534         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2535         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2536         char *block;
2537
2538         switch (mc_id) {
2539         case 32:
2540         case 16:
2541         case 96:
2542         case 80:
2543         case 160:
2544         case 144:
2545         case 224:
2546         case 208:
2547                 block = "CB";
2548                 break;
2549         case 33:
2550         case 17:
2551         case 97:
2552         case 81:
2553         case 161:
2554         case 145:
2555         case 225:
2556         case 209:
2557                 block = "CB_FMASK";
2558                 break;
2559         case 34:
2560         case 18:
2561         case 98:
2562         case 82:
2563         case 162:
2564         case 146:
2565         case 226:
2566         case 210:
2567                 block = "CB_CMASK";
2568                 break;
2569         case 35:
2570         case 19:
2571         case 99:
2572         case 83:
2573         case 163:
2574         case 147:
2575         case 227:
2576         case 211:
2577                 block = "CB_IMMED";
2578                 break;
2579         case 36:
2580         case 20:
2581         case 100:
2582         case 84:
2583         case 164:
2584         case 148:
2585         case 228:
2586         case 212:
2587                 block = "DB";
2588                 break;
2589         case 37:
2590         case 21:
2591         case 101:
2592         case 85:
2593         case 165:
2594         case 149:
2595         case 229:
2596         case 213:
2597                 block = "DB_HTILE";
2598                 break;
2599         case 38:
2600         case 22:
2601         case 102:
2602         case 86:
2603         case 166:
2604         case 150:
2605         case 230:
2606         case 214:
2607                 block = "SX";
2608                 break;
2609         case 39:
2610         case 23:
2611         case 103:
2612         case 87:
2613         case 167:
2614         case 151:
2615         case 231:
2616         case 215:
2617                 block = "DB_STEN";
2618                 break;
2619         case 40:
2620         case 24:
2621         case 104:
2622         case 88:
2623         case 232:
2624         case 216:
2625         case 168:
2626         case 152:
2627                 block = "TC_TFETCH";
2628                 break;
2629         case 41:
2630         case 25:
2631         case 105:
2632         case 89:
2633         case 233:
2634         case 217:
2635         case 169:
2636         case 153:
2637                 block = "TC_VFETCH";
2638                 break;
2639         case 42:
2640         case 26:
2641         case 106:
2642         case 90:
2643         case 234:
2644         case 218:
2645         case 170:
2646         case 154:
2647                 block = "VC";
2648                 break;
2649         case 112:
2650                 block = "CP";
2651                 break;
2652         case 113:
2653         case 114:
2654                 block = "SH";
2655                 break;
2656         case 115:
2657                 block = "VGT";
2658                 break;
2659         case 178:
2660                 block = "IH";
2661                 break;
2662         case 51:
2663                 block = "RLC";
2664                 break;
2665         case 55:
2666                 block = "DMA";
2667                 break;
2668         case 56:
2669                 block = "HDP";
2670                 break;
2671         default:
2672                 block = "unknown";
2673                 break;
2674         }
2675
2676         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2677                protections, vmid, addr,
2678                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2679                block, mc_id);
2680 }
2681
2682 /**
2683  * cayman_vm_flush - vm flush using the CP
2684  *
2685  * @rdev: radeon_device pointer
2686  *
2687  * Update the page table base and flush the VM TLB
2688  * using the CP (cayman-si).
2689  */
2690 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2691                      unsigned vm_id, uint64_t pd_addr)
2692 {
2693         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2694         radeon_ring_write(ring, pd_addr >> 12);
2695
2696         /* flush hdp cache */
2697         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2698         radeon_ring_write(ring, 0x1);
2699
2700         /* bits 0-7 are the VM contexts0-7 */
2701         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2702         radeon_ring_write(ring, 1 << vm_id);
2703
2704         /* wait for the invalidate to complete */
2705         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2706         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2707                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2708         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2709         radeon_ring_write(ring, 0);
2710         radeon_ring_write(ring, 0); /* ref */
2711         radeon_ring_write(ring, 0); /* mask */
2712         radeon_ring_write(ring, 0x20); /* poll interval */
2713
2714         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2715         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2716         radeon_ring_write(ring, 0x0);
2717 }
2718
2719 int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2720 {
2721         struct atom_clock_dividers dividers;
2722         int r, i;
2723
2724         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2725                                            ecclk, false, &dividers);
2726         if (r)
2727                 return r;
2728
2729         for (i = 0; i < 100; i++) {
2730                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2731                         break;
2732                 mdelay(10);
2733         }
2734         if (i == 100)
2735                 return -ETIMEDOUT;
2736
2737         WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2738
2739         for (i = 0; i < 100; i++) {
2740                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2741                         break;
2742                 mdelay(10);
2743         }
2744         if (i == 100)
2745                 return -ETIMEDOUT;
2746
2747         return 0;
2748 }