drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 /*
  40  * Indirect registers accessor
  41  */
  42 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  43 {
  44         unsigned long flags;
  45         u32 r;
  46
  47         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  48         WREG32(TN_SMC_IND_INDEX_0, (reg));
  49         r = RREG32(TN_SMC_IND_DATA_0);
  50         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  51         return r;
  52 }
  53
  54 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  55 {
  56         unsigned long flags;
  57
  58         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  59         WREG32(TN_SMC_IND_INDEX_0, (reg));
  60         WREG32(TN_SMC_IND_DATA_0, (v));
  61         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  62 }
  63
  64 static const u32 tn_rlc_save_restore_register_list[] =
  65 {
  66         0x98fc,
  67         0x98f0,
  68         0x9834,
  69         0x9838,
  70         0x9870,
  71         0x9874,
  72         0x8a14,
  73         0x8b24,
  74         0x8bcc,
  75         0x8b10,
  76         0x8c30,
  77         0x8d00,
  78         0x8d04,
  79         0x8c00,
  80         0x8c04,
  81         0x8c10,
  82         0x8c14,
  83         0x8d8c,
  84         0x8cf0,
  85         0x8e38,
  86         0x9508,
  87         0x9688,
  88         0x9608,
  89         0x960c,
  90         0x9610,
  91         0x9614,
  92         0x88c4,
  93         0x8978,
  94         0x88d4,
  95         0x900c,
  96         0x9100,
  97         0x913c,
  98         0x90e8,
  99         0x9354,
 100         0xa008,
 101         0x98f8,
 102         0x9148,
 103         0x914c,
 104         0x3f94,
 105         0x98f4,
 106         0x9b7c,
 107         0x3f8c,
 108         0x8950,
 109         0x8954,
 110         0x8a18,
 111         0x8b28,
 112         0x9144,
 113         0x3f90,
 114         0x915c,
 115         0x9160,
 116         0x9178,
 117         0x917c,
 118         0x9180,
 119         0x918c,
 120         0x9190,
 121         0x9194,
 122         0x9198,
 123         0x919c,
 124         0x91a8,
 125         0x91ac,
 126         0x91b0,
 127         0x91b4,
 128         0x91b8,
 129         0x91c4,
 130         0x91c8,
 131         0x91cc,
 132         0x91d0,
 133         0x91d4,
 134         0x91e0,
 135         0x91e4,
 136         0x91ec,
 137         0x91f0,
 138         0x91f4,
 139         0x9200,
 140         0x9204,
 141         0x929c,
 142         0x8030,
 143         0x9150,
 144         0x9a60,
 145         0x920c,
 146         0x9210,
 147         0x9228,
 148         0x922c,
 149         0x9244,
 150         0x9248,
 151         0x91e8,
 152         0x9294,
 153         0x9208,
 154         0x9224,
 155         0x9240,
 156         0x9220,
 157         0x923c,
 158         0x9258,
 159         0x9744,
 160         0xa200,
 161         0xa204,
 162         0xa208,
 163         0xa20c,
 164         0x8d58,
 165         0x9030,
 166         0x9034,
 167         0x9038,
 168         0x903c,
 169         0x9040,
 170         0x9654,
 171         0x897c,
 172         0xa210,
 173         0xa214,
 174         0x9868,
 175         0xa02c,
 176         0x9664,
 177         0x9698,
 178         0x949c,
 179         0x8e10,
 180         0x8e18,
 181         0x8c50,
 182         0x8c58,
 183         0x8c60,
 184         0x8c68,
 185         0x89b4,
 186         0x9830,
 187         0x802c,
 188 };
 189
 190 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 191 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 192 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 193 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 194 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 195 extern void evergreen_mc_program(struct radeon_device *rdev);
 196 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 197 extern int evergreen_mc_init(struct radeon_device *rdev);
 198 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 199 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 200 extern void evergreen_program_aspm(struct radeon_device *rdev);
 201 extern void sumo_rlc_fini(struct radeon_device *rdev);
 202 extern int sumo_rlc_init(struct radeon_device *rdev);
 203 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 204
 205 /* Firmware Names */
 206 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 207 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 208 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 209 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 210 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 211 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 212 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 213 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 214 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 215 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 216 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 217 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 218 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 219 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 220 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 221 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 222 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 223 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 224 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 225 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 226 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 227
 228
 229 static const u32 cayman_golden_registers2[] =
 230 {
 231         0x3e5c, 0xffffffff, 0x00000000,
 232         0x3e48, 0xffffffff, 0x00000000,
 233         0x3e4c, 0xffffffff, 0x00000000,
 234         0x3e64, 0xffffffff, 0x00000000,
 235         0x3e50, 0xffffffff, 0x00000000,
 236         0x3e60, 0xffffffff, 0x00000000
 237 };
 238
 239 static const u32 cayman_golden_registers[] =
 240 {
 241         0x5eb4, 0xffffffff, 0x00000002,
 242         0x5e78, 0x8f311ff1, 0x001000f0,
 243         0x3f90, 0xffff0000, 0xff000000,
 244         0x9148, 0xffff0000, 0xff000000,
 245         0x3f94, 0xffff0000, 0xff000000,
 246         0x914c, 0xffff0000, 0xff000000,
 247         0xc78, 0x00000080, 0x00000080,
 248         0xbd4, 0x70073777, 0x00011003,
 249         0xd02c, 0xbfffff1f, 0x08421000,
 250         0xd0b8, 0x73773777, 0x02011003,
 251         0x5bc0, 0x00200000, 0x50100000,
 252         0x98f8, 0x33773777, 0x02011003,
 253         0x98fc, 0xffffffff, 0x76541032,
 254         0x7030, 0x31000311, 0x00000011,
 255         0x2f48, 0x33773777, 0x42010001,
 256         0x6b28, 0x00000010, 0x00000012,
 257         0x7728, 0x00000010, 0x00000012,
 258         0x10328, 0x00000010, 0x00000012,
 259         0x10f28, 0x00000010, 0x00000012,
 260         0x11b28, 0x00000010, 0x00000012,
 261         0x12728, 0x00000010, 0x00000012,
 262         0x240c, 0x000007ff, 0x00000000,
 263         0x8a14, 0xf000001f, 0x00000007,
 264         0x8b24, 0x3fff3fff, 0x00ff0fff,
 265         0x8b10, 0x0000ff0f, 0x00000000,
 266         0x28a4c, 0x07ffffff, 0x06000000,
 267         0x10c, 0x00000001, 0x00010003,
 268         0xa02c, 0xffffffff, 0x0000009b,
 269         0x913c, 0x0000010f, 0x01000100,
 270         0x8c04, 0xf8ff00ff, 0x40600060,
 271         0x28350, 0x00000f01, 0x00000000,
 272         0x9508, 0x3700001f, 0x00000002,
 273         0x960c, 0xffffffff, 0x54763210,
 274         0x88c4, 0x001f3ae3, 0x00000082,
 275         0x88d0, 0xffffffff, 0x0f40df40,
 276         0x88d4, 0x0000001f, 0x00000010,
 277         0x8974, 0xffffffff, 0x00000000
 278 };
 279
 280 static const u32 dvst_golden_registers2[] =
 281 {
 282         0x8f8, 0xffffffff, 0,
 283         0x8fc, 0x00380000, 0,
 284         0x8f8, 0xffffffff, 1,
 285         0x8fc, 0x0e000000, 0
 286 };
 287
 288 static const u32 dvst_golden_registers[] =
 289 {
 290         0x690, 0x3fff3fff, 0x20c00033,
 291         0x918c, 0x0fff0fff, 0x00010006,
 292         0x91a8, 0x0fff0fff, 0x00010006,
 293         0x9150, 0xffffdfff, 0x6e944040,
 294         0x917c, 0x0fff0fff, 0x00030002,
 295         0x9198, 0x0fff0fff, 0x00030002,
 296         0x915c, 0x0fff0fff, 0x00010000,
 297         0x3f90, 0xffff0001, 0xff000000,
 298         0x9178, 0x0fff0fff, 0x00070000,
 299         0x9194, 0x0fff0fff, 0x00070000,
 300         0x9148, 0xffff0001, 0xff000000,
 301         0x9190, 0x0fff0fff, 0x00090008,
 302         0x91ac, 0x0fff0fff, 0x00090008,
 303         0x3f94, 0xffff0000, 0xff000000,
 304         0x914c, 0xffff0000, 0xff000000,
 305         0x929c, 0x00000fff, 0x00000001,
 306         0x55e4, 0xff607fff, 0xfc000100,
 307         0x8a18, 0xff000fff, 0x00000100,
 308         0x8b28, 0xff000fff, 0x00000100,
 309         0x9144, 0xfffc0fff, 0x00000100,
 310         0x6ed8, 0x00010101, 0x00010000,
 311         0x9830, 0xffffffff, 0x00000000,
 312         0x9834, 0xf00fffff, 0x00000400,
 313         0x9838, 0xfffffffe, 0x00000000,
 314         0xd0c0, 0xff000fff, 0x00000100,
 315         0xd02c, 0xbfffff1f, 0x08421000,
 316         0xd0b8, 0x73773777, 0x12010001,
 317         0x5bb0, 0x000000f0, 0x00000070,
 318         0x98f8, 0x73773777, 0x12010001,
 319         0x98fc, 0xffffffff, 0x00000010,
 320         0x9b7c, 0x00ff0000, 0x00fc0000,
 321         0x8030, 0x00001f0f, 0x0000100a,
 322         0x2f48, 0x73773777, 0x12010001,
 323         0x2408, 0x00030000, 0x000c007f,
 324         0x8a14, 0xf000003f, 0x00000007,
 325         0x8b24, 0x3fff3fff, 0x00ff0fff,
 326         0x8b10, 0x0000ff0f, 0x00000000,
 327         0x28a4c, 0x07ffffff, 0x06000000,
 328         0x4d8, 0x00000fff, 0x00000100,
 329         0xa008, 0xffffffff, 0x00010000,
 330         0x913c, 0xffff03ff, 0x01000100,
 331         0x8c00, 0x000000ff, 0x00000003,
 332         0x8c04, 0xf8ff00ff, 0x40600060,
 333         0x8cf0, 0x1fff1fff, 0x08e00410,
 334         0x28350, 0x00000f01, 0x00000000,
 335         0x9508, 0xf700071f, 0x00000002,
 336         0x960c, 0xffffffff, 0x54763210,
 337         0x20ef8, 0x01ff01ff, 0x00000002,
 338         0x20e98, 0xfffffbff, 0x00200000,
 339         0x2015c, 0xffffffff, 0x00000f40,
 340         0x88c4, 0x001f3ae3, 0x00000082,
 341         0x8978, 0x3fffffff, 0x04050140,
 342         0x88d4, 0x0000001f, 0x00000010,
 343         0x8974, 0xffffffff, 0x00000000
 344 };
 345
 346 static const u32 scrapper_golden_registers[] =
 347 {
 348         0x690, 0x3fff3fff, 0x20c00033,
 349         0x918c, 0x0fff0fff, 0x00010006,
 350         0x918c, 0x0fff0fff, 0x00010006,
 351         0x91a8, 0x0fff0fff, 0x00010006,
 352         0x91a8, 0x0fff0fff, 0x00010006,
 353         0x9150, 0xffffdfff, 0x6e944040,
 354         0x9150, 0xffffdfff, 0x6e944040,
 355         0x917c, 0x0fff0fff, 0x00030002,
 356         0x917c, 0x0fff0fff, 0x00030002,
 357         0x9198, 0x0fff0fff, 0x00030002,
 358         0x9198, 0x0fff0fff, 0x00030002,
 359         0x915c, 0x0fff0fff, 0x00010000,
 360         0x915c, 0x0fff0fff, 0x00010000,
 361         0x3f90, 0xffff0001, 0xff000000,
 362         0x3f90, 0xffff0001, 0xff000000,
 363         0x9178, 0x0fff0fff, 0x00070000,
 364         0x9178, 0x0fff0fff, 0x00070000,
 365         0x9194, 0x0fff0fff, 0x00070000,
 366         0x9194, 0x0fff0fff, 0x00070000,
 367         0x9148, 0xffff0001, 0xff000000,
 368         0x9148, 0xffff0001, 0xff000000,
 369         0x9190, 0x0fff0fff, 0x00090008,
 370         0x9190, 0x0fff0fff, 0x00090008,
 371         0x91ac, 0x0fff0fff, 0x00090008,
 372         0x91ac, 0x0fff0fff, 0x00090008,
 373         0x3f94, 0xffff0000, 0xff000000,
 374         0x3f94, 0xffff0000, 0xff000000,
 375         0x914c, 0xffff0000, 0xff000000,
 376         0x914c, 0xffff0000, 0xff000000,
 377         0x929c, 0x00000fff, 0x00000001,
 378         0x929c, 0x00000fff, 0x00000001,
 379         0x55e4, 0xff607fff, 0xfc000100,
 380         0x8a18, 0xff000fff, 0x00000100,
 381         0x8a18, 0xff000fff, 0x00000100,
 382         0x8b28, 0xff000fff, 0x00000100,
 383         0x8b28, 0xff000fff, 0x00000100,
 384         0x9144, 0xfffc0fff, 0x00000100,
 385         0x9144, 0xfffc0fff, 0x00000100,
 386         0x6ed8, 0x00010101, 0x00010000,
 387         0x9830, 0xffffffff, 0x00000000,
 388         0x9830, 0xffffffff, 0x00000000,
 389         0x9834, 0xf00fffff, 0x00000400,
 390         0x9834, 0xf00fffff, 0x00000400,
 391         0x9838, 0xfffffffe, 0x00000000,
 392         0x9838, 0xfffffffe, 0x00000000,
 393         0xd0c0, 0xff000fff, 0x00000100,
 394         0xd02c, 0xbfffff1f, 0x08421000,
 395         0xd02c, 0xbfffff1f, 0x08421000,
 396         0xd0b8, 0x73773777, 0x12010001,
 397         0xd0b8, 0x73773777, 0x12010001,
 398         0x5bb0, 0x000000f0, 0x00000070,
 399         0x98f8, 0x73773777, 0x12010001,
 400         0x98f8, 0x73773777, 0x12010001,
 401         0x98fc, 0xffffffff, 0x00000010,
 402         0x98fc, 0xffffffff, 0x00000010,
 403         0x9b7c, 0x00ff0000, 0x00fc0000,
 404         0x9b7c, 0x00ff0000, 0x00fc0000,
 405         0x8030, 0x00001f0f, 0x0000100a,
 406         0x8030, 0x00001f0f, 0x0000100a,
 407         0x2f48, 0x73773777, 0x12010001,
 408         0x2f48, 0x73773777, 0x12010001,
 409         0x2408, 0x00030000, 0x000c007f,
 410         0x8a14, 0xf000003f, 0x00000007,
 411         0x8a14, 0xf000003f, 0x00000007,
 412         0x8b24, 0x3fff3fff, 0x00ff0fff,
 413         0x8b24, 0x3fff3fff, 0x00ff0fff,
 414         0x8b10, 0x0000ff0f, 0x00000000,
 415         0x8b10, 0x0000ff0f, 0x00000000,
 416         0x28a4c, 0x07ffffff, 0x06000000,
 417         0x28a4c, 0x07ffffff, 0x06000000,
 418         0x4d8, 0x00000fff, 0x00000100,
 419         0x4d8, 0x00000fff, 0x00000100,
 420         0xa008, 0xffffffff, 0x00010000,
 421         0xa008, 0xffffffff, 0x00010000,
 422         0x913c, 0xffff03ff, 0x01000100,
 423         0x913c, 0xffff03ff, 0x01000100,
 424         0x90e8, 0x001fffff, 0x010400c0,
 425         0x8c00, 0x000000ff, 0x00000003,
 426         0x8c00, 0x000000ff, 0x00000003,
 427         0x8c04, 0xf8ff00ff, 0x40600060,
 428         0x8c04, 0xf8ff00ff, 0x40600060,
 429         0x8c30, 0x0000000f, 0x00040005,
 430         0x8cf0, 0x1fff1fff, 0x08e00410,
 431         0x8cf0, 0x1fff1fff, 0x08e00410,
 432         0x900c, 0x00ffffff, 0x0017071f,
 433         0x28350, 0x00000f01, 0x00000000,
 434         0x28350, 0x00000f01, 0x00000000,
 435         0x9508, 0xf700071f, 0x00000002,
 436         0x9508, 0xf700071f, 0x00000002,
 437         0x9688, 0x00300000, 0x0017000f,
 438         0x960c, 0xffffffff, 0x54763210,
 439         0x960c, 0xffffffff, 0x54763210,
 440         0x20ef8, 0x01ff01ff, 0x00000002,
 441         0x20e98, 0xfffffbff, 0x00200000,
 442         0x2015c, 0xffffffff, 0x00000f40,
 443         0x88c4, 0x001f3ae3, 0x00000082,
 444         0x88c4, 0x001f3ae3, 0x00000082,
 445         0x8978, 0x3fffffff, 0x04050140,
 446         0x8978, 0x3fffffff, 0x04050140,
 447         0x88d4, 0x0000001f, 0x00000010,
 448         0x88d4, 0x0000001f, 0x00000010,
 449         0x8974, 0xffffffff, 0x00000000,
 450         0x8974, 0xffffffff, 0x00000000
 451 };
 452
 453 static void ni_init_golden_registers(struct radeon_device *rdev)
 454 {
 455         switch (rdev->family) {
 456         case CHIP_CAYMAN:
 457                 radeon_program_register_sequence(rdev,
 458                                                  cayman_golden_registers,
 459                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 460                 radeon_program_register_sequence(rdev,
 461                                                  cayman_golden_registers2,
 462                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 463                 break;
 464         case CHIP_ARUBA:
 465                 if ((rdev->pdev->device == 0x9900) ||
 466                     (rdev->pdev->device == 0x9901) ||
 467                     (rdev->pdev->device == 0x9903) ||
 468                     (rdev->pdev->device == 0x9904) ||
 469                     (rdev->pdev->device == 0x9905) ||
 470                     (rdev->pdev->device == 0x9906) ||
 471                     (rdev->pdev->device == 0x9907) ||
 472                     (rdev->pdev->device == 0x9908) ||
 473                     (rdev->pdev->device == 0x9909) ||
 474                     (rdev->pdev->device == 0x990A) ||
 475                     (rdev->pdev->device == 0x990B) ||
 476                     (rdev->pdev->device == 0x990C) ||
 477                     (rdev->pdev->device == 0x990D) ||
 478                     (rdev->pdev->device == 0x990E) ||
 479                     (rdev->pdev->device == 0x990F) ||
 480                     (rdev->pdev->device == 0x9910) ||
 481                     (rdev->pdev->device == 0x9913) ||
 482                     (rdev->pdev->device == 0x9917) ||
 483                     (rdev->pdev->device == 0x9918)) {
 484                         radeon_program_register_sequence(rdev,
 485                                                          dvst_golden_registers,
 486                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 487                         radeon_program_register_sequence(rdev,
 488                                                          dvst_golden_registers2,
 489                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 490                 } else {
 491                         radeon_program_register_sequence(rdev,
 492                                                          scrapper_golden_registers,
 493                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 494                         radeon_program_register_sequence(rdev,
 495                                                          dvst_golden_registers2,
 496                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 497                 }
 498                 break;
 499         default:
 500                 break;
 501         }
 502 }
 503
 504 #define BTC_IO_MC_REGS_SIZE 29
 505
 506 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 507         {0x00000077, 0xff010100},
 508         {0x00000078, 0x00000000},
 509         {0x00000079, 0x00001434},
 510         {0x0000007a, 0xcc08ec08},
 511         {0x0000007b, 0x00040000},
 512         {0x0000007c, 0x000080c0},
 513         {0x0000007d, 0x09000000},
 514         {0x0000007e, 0x00210404},
 515         {0x00000081, 0x08a8e800},
 516         {0x00000082, 0x00030444},
 517         {0x00000083, 0x00000000},
 518         {0x00000085, 0x00000001},
 519         {0x00000086, 0x00000002},
 520         {0x00000087, 0x48490000},
 521         {0x00000088, 0x20244647},
 522         {0x00000089, 0x00000005},
 523         {0x0000008b, 0x66030000},
 524         {0x0000008c, 0x00006603},
 525         {0x0000008d, 0x00000100},
 526         {0x0000008f, 0x00001c0a},
 527         {0x00000090, 0xff000001},
 528         {0x00000094, 0x00101101},
 529         {0x00000095, 0x00000fff},
 530         {0x00000096, 0x00116fff},
 531         {0x00000097, 0x60010000},
 532         {0x00000098, 0x10010000},
 533         {0x00000099, 0x00006000},
 534         {0x0000009a, 0x00001000},
 535         {0x0000009f, 0x00946a00}
 536 };
 537
 538 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 539         {0x00000077, 0xff010100},
 540         {0x00000078, 0x00000000},
 541         {0x00000079, 0x00001434},
 542         {0x0000007a, 0xcc08ec08},
 543         {0x0000007b, 0x00040000},
 544         {0x0000007c, 0x000080c0},
 545         {0x0000007d, 0x09000000},
 546         {0x0000007e, 0x00210404},
 547         {0x00000081, 0x08a8e800},
 548         {0x00000082, 0x00030444},
 549         {0x00000083, 0x00000000},
 550         {0x00000085, 0x00000001},
 551         {0x00000086, 0x00000002},
 552         {0x00000087, 0x48490000},
 553         {0x00000088, 0x20244647},
 554         {0x00000089, 0x00000005},
 555         {0x0000008b, 0x66030000},
 556         {0x0000008c, 0x00006603},
 557         {0x0000008d, 0x00000100},
 558         {0x0000008f, 0x00001c0a},
 559         {0x00000090, 0xff000001},
 560         {0x00000094, 0x00101101},
 561         {0x00000095, 0x00000fff},
 562         {0x00000096, 0x00116fff},
 563         {0x00000097, 0x60010000},
 564         {0x00000098, 0x10010000},
 565         {0x00000099, 0x00006000},
 566         {0x0000009a, 0x00001000},
 567         {0x0000009f, 0x00936a00}
 568 };
 569
 570 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 571         {0x00000077, 0xff010100},
 572         {0x00000078, 0x00000000},
 573         {0x00000079, 0x00001434},
 574         {0x0000007a, 0xcc08ec08},
 575         {0x0000007b, 0x00040000},
 576         {0x0000007c, 0x000080c0},
 577         {0x0000007d, 0x09000000},
 578         {0x0000007e, 0x00210404},
 579         {0x00000081, 0x08a8e800},
 580         {0x00000082, 0x00030444},
 581         {0x00000083, 0x00000000},
 582         {0x00000085, 0x00000001},
 583         {0x00000086, 0x00000002},
 584         {0x00000087, 0x48490000},
 585         {0x00000088, 0x20244647},
 586         {0x00000089, 0x00000005},
 587         {0x0000008b, 0x66030000},
 588         {0x0000008c, 0x00006603},
 589         {0x0000008d, 0x00000100},
 590         {0x0000008f, 0x00001c0a},
 591         {0x00000090, 0xff000001},
 592         {0x00000094, 0x00101101},
 593         {0x00000095, 0x00000fff},
 594         {0x00000096, 0x00116fff},
 595         {0x00000097, 0x60010000},
 596         {0x00000098, 0x10010000},
 597         {0x00000099, 0x00006000},
 598         {0x0000009a, 0x00001000},
 599         {0x0000009f, 0x00916a00}
 600 };
 601
 602 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 603         {0x00000077, 0xff010100},
 604         {0x00000078, 0x00000000},
 605         {0x00000079, 0x00001434},
 606         {0x0000007a, 0xcc08ec08},
 607         {0x0000007b, 0x00040000},
 608         {0x0000007c, 0x000080c0},
 609         {0x0000007d, 0x09000000},
 610         {0x0000007e, 0x00210404},
 611         {0x00000081, 0x08a8e800},
 612         {0x00000082, 0x00030444},
 613         {0x00000083, 0x00000000},
 614         {0x00000085, 0x00000001},
 615         {0x00000086, 0x00000002},
 616         {0x00000087, 0x48490000},
 617         {0x00000088, 0x20244647},
 618         {0x00000089, 0x00000005},
 619         {0x0000008b, 0x66030000},
 620         {0x0000008c, 0x00006603},
 621         {0x0000008d, 0x00000100},
 622         {0x0000008f, 0x00001c0a},
 623         {0x00000090, 0xff000001},
 624         {0x00000094, 0x00101101},
 625         {0x00000095, 0x00000fff},
 626         {0x00000096, 0x00116fff},
 627         {0x00000097, 0x60010000},
 628         {0x00000098, 0x10010000},
 629         {0x00000099, 0x00006000},
 630         {0x0000009a, 0x00001000},
 631         {0x0000009f, 0x00976b00}
 632 };
 633
 634 int ni_mc_load_microcode(struct radeon_device *rdev)
 635 {
 636         const __be32 *fw_data;
 637         u32 mem_type, running, blackout = 0;
 638         u32 *io_mc_regs;
 639         int i, ucode_size, regs_size;
 640
 641         if (!rdev->mc_fw)
 642                 return -EINVAL;
 643
 644         switch (rdev->family) {
 645         case CHIP_BARTS:
 646                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 647                 ucode_size = BTC_MC_UCODE_SIZE;
 648                 regs_size = BTC_IO_MC_REGS_SIZE;
 649                 break;
 650         case CHIP_TURKS:
 651                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 652                 ucode_size = BTC_MC_UCODE_SIZE;
 653                 regs_size = BTC_IO_MC_REGS_SIZE;
 654                 break;
 655         case CHIP_CAICOS:
 656         default:
 657                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 658                 ucode_size = BTC_MC_UCODE_SIZE;
 659                 regs_size = BTC_IO_MC_REGS_SIZE;
 660                 break;
 661         case CHIP_CAYMAN:
 662                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 663                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 664                 regs_size = BTC_IO_MC_REGS_SIZE;
 665                 break;
 666         }
 667
 668         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 669         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 670
 671         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 672                 if (running) {
 673                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 674                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 675                 }
 676
 677                 /* reset the engine and set to writable */
 678                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 679                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 680
 681                 /* load mc io regs */
 682                 for (i = 0; i < regs_size; i++) {
 683                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 684                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 685                 }
 686                 /* load the MC ucode */
 687                 fw_data = (const __be32 *)rdev->mc_fw->data;
 688                 for (i = 0; i < ucode_size; i++)
 689                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 690
 691                 /* put the engine back into the active state */
 692                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 693                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 694                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 695
 696                 /* wait for training to complete */
 697                 for (i = 0; i < rdev->usec_timeout; i++) {
 698                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 699                                 break;
 700                         udelay(1);
 701                 }
 702
 703                 if (running)
 704                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 705         }
 706
 707         return 0;
 708 }
 709
 710 int ni_init_microcode(struct radeon_device *rdev)
 711 {
 712         const char *chip_name;
 713         const char *rlc_chip_name;
 714         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 715         size_t smc_req_size = 0;
 716         char fw_name[30];
 717         int err;
 718
 719         DRM_DEBUG("\n");
 720
 721         switch (rdev->family) {
 722         case CHIP_BARTS:
 723                 chip_name = "BARTS";
 724                 rlc_chip_name = "BTC";
 725                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 726                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 727                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 728                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 729                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 730                 break;
 731         case CHIP_TURKS:
 732                 chip_name = "TURKS";
 733                 rlc_chip_name = "BTC";
 734                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 735                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 736                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 737                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 738                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 739                 break;
 740         case CHIP_CAICOS:
 741                 chip_name = "CAICOS";
 742                 rlc_chip_name = "BTC";
 743                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 744                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 745                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 746                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 747                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 748                 break;
 749         case CHIP_CAYMAN:
 750                 chip_name = "CAYMAN";
 751                 rlc_chip_name = "CAYMAN";
 752                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 753                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 754                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 755                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 756                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 757                 break;
 758         case CHIP_ARUBA:
 759                 chip_name = "ARUBA";
 760                 rlc_chip_name = "ARUBA";
 761                 /* pfp/me same size as CAYMAN */
 762                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 763                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 764                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 765                 mc_req_size = 0;
 766                 break;
 767         default: BUG();
 768         }
 769
 770         DRM_INFO("Loading %s Microcode\n", chip_name);
 771
 772         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 773         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 774         if (err)
 775                 goto out;
 776         if (rdev->pfp_fw->size != pfp_req_size) {
 777                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 778                        rdev->pfp_fw->size, fw_name);
 779                 err = -EINVAL;
 780                 goto out;
 781         }
 782
 783         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 784         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 785         if (err)
 786                 goto out;
 787         if (rdev->me_fw->size != me_req_size) {
 788                 pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 789                        rdev->me_fw->size, fw_name);
 790                 err = -EINVAL;
 791         }
 792
 793         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 794         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 795         if (err)
 796                 goto out;
 797         if (rdev->rlc_fw->size != rlc_req_size) {
 798                 pr_err("ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 799                        rdev->rlc_fw->size, fw_name);
 800                 err = -EINVAL;
 801         }
 802
 803         /* no MC ucode on TN */
 804         if (!(rdev->flags & RADEON_IS_IGP)) {
 805                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 806                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 807                 if (err)
 808                         goto out;
 809                 if (rdev->mc_fw->size != mc_req_size) {
 810                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 811                                rdev->mc_fw->size, fw_name);
 812                         err = -EINVAL;
 813                 }
 814         }
 815
 816         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 817                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 818                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 819                 if (err) {
 820                         pr_err("smc: error loading firmware \"%s\"\n", fw_name);
 821                         release_firmware(rdev->smc_fw);
 822                         rdev->smc_fw = NULL;
 823                         err = 0;
 824                 } else if (rdev->smc_fw->size != smc_req_size) {
 825                         pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 826                                rdev->mc_fw->size, fw_name);
 827                         err = -EINVAL;
 828                 }
 829         }
 830
 831 out:
 832         if (err) {
 833                 if (err != -EINVAL)
 834                         pr_err("ni_cp: Failed to load firmware \"%s\"\n",
 835                                fw_name);
 836                 release_firmware(rdev->pfp_fw);
 837                 rdev->pfp_fw = NULL;
 838                 release_firmware(rdev->me_fw);
 839                 rdev->me_fw = NULL;
 840                 release_firmware(rdev->rlc_fw);
 841                 rdev->rlc_fw = NULL;
 842                 release_firmware(rdev->mc_fw);
 843                 rdev->mc_fw = NULL;
 844         }
 845         return err;
 846 }
 847
 848 /**
 849  * cayman_get_allowed_info_register - fetch the register for the info ioctl
 850  *
 851  * @rdev: radeon_device pointer
 852  * @reg: register offset in bytes
 853  * @val: register value
 854  *
 855  * Returns 0 for success or -EINVAL for an invalid register
 856  *
 857  */
 858 int cayman_get_allowed_info_register(struct radeon_device *rdev,
 859                                      u32 reg, u32 *val)
 860 {
 861         switch (reg) {
 862         case GRBM_STATUS:
 863         case GRBM_STATUS_SE0:
 864         case GRBM_STATUS_SE1:
 865         case SRBM_STATUS:
 866         case SRBM_STATUS2:
 867         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 868         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 869         case UVD_STATUS:
 870                 *val = RREG32(reg);
 871                 return 0;
 872         default:
 873                 return -EINVAL;
 874         }
 875 }
 876
 877 int tn_get_temp(struct radeon_device *rdev)
 878 {
 879         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 880         int actual_temp = (temp / 8) - 49;
 881
 882         return actual_temp * 1000;
 883 }
 884
 885 /*
 886  * Core functions
 887  */
 888 static void cayman_gpu_init(struct radeon_device *rdev)
 889 {
 890         u32 gb_addr_config = 0;
 891         u32 mc_shared_chmap, mc_arb_ramcfg;
 892         u32 cgts_tcc_disable;
 893         u32 sx_debug_1;
 894         u32 smx_dc_ctl0;
 895         u32 cgts_sm_ctrl_reg;
 896         u32 hdp_host_path_cntl;
 897         u32 tmp;
 898         u32 disabled_rb_mask;
 899         int i, j;
 900
 901         switch (rdev->family) {
 902         case CHIP_CAYMAN:
 903                 rdev->config.cayman.max_shader_engines = 2;
 904                 rdev->config.cayman.max_pipes_per_simd = 4;
 905                 rdev->config.cayman.max_tile_pipes = 8;
 906                 rdev->config.cayman.max_simds_per_se = 12;
 907                 rdev->config.cayman.max_backends_per_se = 4;
 908                 rdev->config.cayman.max_texture_channel_caches = 8;
 909                 rdev->config.cayman.max_gprs = 256;
 910                 rdev->config.cayman.max_threads = 256;
 911                 rdev->config.cayman.max_gs_threads = 32;
 912                 rdev->config.cayman.max_stack_entries = 512;
 913                 rdev->config.cayman.sx_num_of_sets = 8;
 914                 rdev->config.cayman.sx_max_export_size = 256;
 915                 rdev->config.cayman.sx_max_export_pos_size = 64;
 916                 rdev->config.cayman.sx_max_export_smx_size = 192;
 917                 rdev->config.cayman.max_hw_contexts = 8;
 918                 rdev->config.cayman.sq_num_cf_insts = 2;
 919
 920                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 921                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 922                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 923                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 924                 break;
 925         case CHIP_ARUBA:
 926         default:
 927                 rdev->config.cayman.max_shader_engines = 1;
 928                 rdev->config.cayman.max_pipes_per_simd = 4;
 929                 rdev->config.cayman.max_tile_pipes = 2;
 930                 if ((rdev->pdev->device == 0x9900) ||
 931                     (rdev->pdev->device == 0x9901) ||
 932                     (rdev->pdev->device == 0x9905) ||
 933                     (rdev->pdev->device == 0x9906) ||
 934                     (rdev->pdev->device == 0x9907) ||
 935                     (rdev->pdev->device == 0x9908) ||
 936                     (rdev->pdev->device == 0x9909) ||
 937                     (rdev->pdev->device == 0x990B) ||
 938                     (rdev->pdev->device == 0x990C) ||
 939                     (rdev->pdev->device == 0x990F) ||
 940                     (rdev->pdev->device == 0x9910) ||
 941                     (rdev->pdev->device == 0x9917) ||
 942                     (rdev->pdev->device == 0x9999) ||
 943                     (rdev->pdev->device == 0x999C)) {
 944                         rdev->config.cayman.max_simds_per_se = 6;
 945                         rdev->config.cayman.max_backends_per_se = 2;
 946                         rdev->config.cayman.max_hw_contexts = 8;
 947                         rdev->config.cayman.sx_max_export_size = 256;
 948                         rdev->config.cayman.sx_max_export_pos_size = 64;
 949                         rdev->config.cayman.sx_max_export_smx_size = 192;
 950                 } else if ((rdev->pdev->device == 0x9903) ||
 951                            (rdev->pdev->device == 0x9904) ||
 952                            (rdev->pdev->device == 0x990A) ||
 953                            (rdev->pdev->device == 0x990D) ||
 954                            (rdev->pdev->device == 0x990E) ||
 955                            (rdev->pdev->device == 0x9913) ||
 956                            (rdev->pdev->device == 0x9918) ||
 957                            (rdev->pdev->device == 0x999D)) {
 958                         rdev->config.cayman.max_simds_per_se = 4;
 959                         rdev->config.cayman.max_backends_per_se = 2;
 960                         rdev->config.cayman.max_hw_contexts = 8;
 961                         rdev->config.cayman.sx_max_export_size = 256;
 962                         rdev->config.cayman.sx_max_export_pos_size = 64;
 963                         rdev->config.cayman.sx_max_export_smx_size = 192;
 964                 } else if ((rdev->pdev->device == 0x9919) ||
 965                            (rdev->pdev->device == 0x9990) ||
 966                            (rdev->pdev->device == 0x9991) ||
 967                            (rdev->pdev->device == 0x9994) ||
 968                            (rdev->pdev->device == 0x9995) ||
 969                            (rdev->pdev->device == 0x9996) ||
 970                            (rdev->pdev->device == 0x999A) ||
 971                            (rdev->pdev->device == 0x99A0)) {
 972                         rdev->config.cayman.max_simds_per_se = 3;
 973                         rdev->config.cayman.max_backends_per_se = 1;
 974                         rdev->config.cayman.max_hw_contexts = 4;
 975                         rdev->config.cayman.sx_max_export_size = 128;
 976                         rdev->config.cayman.sx_max_export_pos_size = 32;
 977                         rdev->config.cayman.sx_max_export_smx_size = 96;
 978                 } else {
 979                         rdev->config.cayman.max_simds_per_se = 2;
 980                         rdev->config.cayman.max_backends_per_se = 1;
 981                         rdev->config.cayman.max_hw_contexts = 4;
 982                         rdev->config.cayman.sx_max_export_size = 128;
 983                         rdev->config.cayman.sx_max_export_pos_size = 32;
 984                         rdev->config.cayman.sx_max_export_smx_size = 96;
 985                 }
 986                 rdev->config.cayman.max_texture_channel_caches = 2;
 987                 rdev->config.cayman.max_gprs = 256;
 988                 rdev->config.cayman.max_threads = 256;
 989                 rdev->config.cayman.max_gs_threads = 32;
 990                 rdev->config.cayman.max_stack_entries = 512;
 991                 rdev->config.cayman.sx_num_of_sets = 8;
 992                 rdev->config.cayman.sq_num_cf_insts = 2;
 993
 994                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 995                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 996                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 997                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 998                 break;
 999         }
1000
1001         /* Initialize HDP */
1002         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1003                 WREG32((0x2c14 + j), 0x00000000);
1004                 WREG32((0x2c18 + j), 0x00000000);
1005                 WREG32((0x2c1c + j), 0x00000000);
1006                 WREG32((0x2c20 + j), 0x00000000);
1007                 WREG32((0x2c24 + j), 0x00000000);
1008         }
1009
1010         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1011         WREG32(SRBM_INT_CNTL, 0x1);
1012         WREG32(SRBM_INT_ACK, 0x1);
1013
1014         evergreen_fix_pci_max_read_req_size(rdev);
1015
1016         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1017         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1018
1019         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1020         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1021         if (rdev->config.cayman.mem_row_size_in_kb > 4)
1022                 rdev->config.cayman.mem_row_size_in_kb = 4;
1023         /* XXX use MC settings? */
1024         rdev->config.cayman.shader_engine_tile_size = 32;
1025         rdev->config.cayman.num_gpus = 1;
1026         rdev->config.cayman.multi_gpu_tile_size = 64;
1027
1028         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1029         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1030         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1031         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1032         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1033         rdev->config.cayman.num_shader_engines = tmp + 1;
1034         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1035         rdev->config.cayman.num_gpus = tmp + 1;
1036         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1037         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1038         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1039         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1040
1041
1042         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1043          * not have bank info, so create a custom tiling dword.
1044          * bits 3:0   num_pipes
1045          * bits 7:4   num_banks
1046          * bits 11:8  group_size
1047          * bits 15:12 row_size
1048          */
1049         rdev->config.cayman.tile_config = 0;
1050         switch (rdev->config.cayman.num_tile_pipes) {
1051         case 1:
1052         default:
1053                 rdev->config.cayman.tile_config |= (0 << 0);
1054                 break;
1055         case 2:
1056                 rdev->config.cayman.tile_config |= (1 << 0);
1057                 break;
1058         case 4:
1059                 rdev->config.cayman.tile_config |= (2 << 0);
1060                 break;
1061         case 8:
1062                 rdev->config.cayman.tile_config |= (3 << 0);
1063                 break;
1064         }
1065
1066         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1067         if (rdev->flags & RADEON_IS_IGP)
1068                 rdev->config.cayman.tile_config |= 1 << 4;
1069         else {
1070                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1071                 case 0: /* four banks */
1072                         rdev->config.cayman.tile_config |= 0 << 4;
1073                         break;
1074                 case 1: /* eight banks */
1075                         rdev->config.cayman.tile_config |= 1 << 4;
1076                         break;
1077                 case 2: /* sixteen banks */
1078                 default:
1079                         rdev->config.cayman.tile_config |= 2 << 4;
1080                         break;
1081                 }
1082         }
1083         rdev->config.cayman.tile_config |=
1084                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1085         rdev->config.cayman.tile_config |=
1086                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1087
1088         tmp = 0;
1089         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1090                 u32 rb_disable_bitmap;
1091
1092                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1093                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1094                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1095                 tmp <<= 4;
1096                 tmp |= rb_disable_bitmap;
1097         }
1098         /* enabled rb are just the one not disabled :) */
1099         disabled_rb_mask = tmp;
1100         tmp = 0;
1101         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1102                 tmp |= (1 << i);
1103         /* if all the backends are disabled, fix it up here */
1104         if ((disabled_rb_mask & tmp) == tmp) {
1105                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1106                         disabled_rb_mask &= ~(1 << i);
1107         }
1108
1109         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1110                 u32 simd_disable_bitmap;
1111
1112                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1113                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1114                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1115                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1116                 tmp <<= 16;
1117                 tmp |= simd_disable_bitmap;
1118         }
1119         rdev->config.cayman.active_simds = hweight32(~tmp);
1120
1121         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1122         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1123
1124         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1125         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1126         if (ASIC_IS_DCE6(rdev))
1127                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1128         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1129         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1130         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1131         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1132         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1133         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1134
1135         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1136             (rdev->flags & RADEON_IS_IGP)) {
1137                 if ((disabled_rb_mask & 3) == 2) {
1138                         /* RB1 disabled, RB0 enabled */
1139                         tmp = 0x00000000;
1140                 } else {
1141                         /* RB0 disabled, RB1 enabled */
1142                         tmp = 0x11111111;
1143                 }
1144         } else {
1145                 tmp = gb_addr_config & NUM_PIPES_MASK;
1146                 tmp = r6xx_remap_render_backend(rdev, tmp,
1147                                                 rdev->config.cayman.max_backends_per_se *
1148                                                 rdev->config.cayman.max_shader_engines,
1149                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1150         }
1151         WREG32(GB_BACKEND_MAP, tmp);
1152
1153         cgts_tcc_disable = 0xffff0000;
1154         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1155                 cgts_tcc_disable &= ~(1 << (16 + i));
1156         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1157         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1158         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1159         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1160
1161         /* reprogram the shader complex */
1162         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1163         for (i = 0; i < 16; i++)
1164                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1165         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1166
1167         /* set HW defaults for 3D engine */
1168         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1169
1170         sx_debug_1 = RREG32(SX_DEBUG_1);
1171         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1172         WREG32(SX_DEBUG_1, sx_debug_1);
1173
1174         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1175         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1176         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1177         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1178
1179         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1180
1181         /* need to be explicitly zero-ed */
1182         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1183         WREG32(SQ_LSTMP_RING_BASE, 0);
1184         WREG32(SQ_HSTMP_RING_BASE, 0);
1185         WREG32(SQ_ESTMP_RING_BASE, 0);
1186         WREG32(SQ_GSTMP_RING_BASE, 0);
1187         WREG32(SQ_VSTMP_RING_BASE, 0);
1188         WREG32(SQ_PSTMP_RING_BASE, 0);
1189
1190         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1191
1192         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1193                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1194                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1195
1196         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1197                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1198                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1199
1200
1201         WREG32(VGT_NUM_INSTANCES, 1);
1202
1203         WREG32(CP_PERFMON_CNTL, 0);
1204
1205         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1206                                   FETCH_FIFO_HIWATER(0x4) |
1207                                   DONE_FIFO_HIWATER(0xe0) |
1208                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1209
1210         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1211         WREG32(SQ_CONFIG, (VC_ENABLE |
1212                            EXPORT_SRC_C |
1213                            GFX_PRIO(0) |
1214                            CS1_PRIO(0) |
1215                            CS2_PRIO(1)));
1216         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1217
1218         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1219                                           FORCE_EOV_MAX_REZ_CNT(255)));
1220
1221         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1222                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1223
1224         WREG32(VGT_GS_VERTEX_REUSE, 16);
1225         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1226
1227         WREG32(CB_PERF_CTR0_SEL_0, 0);
1228         WREG32(CB_PERF_CTR0_SEL_1, 0);
1229         WREG32(CB_PERF_CTR1_SEL_0, 0);
1230         WREG32(CB_PERF_CTR1_SEL_1, 0);
1231         WREG32(CB_PERF_CTR2_SEL_0, 0);
1232         WREG32(CB_PERF_CTR2_SEL_1, 0);
1233         WREG32(CB_PERF_CTR3_SEL_0, 0);
1234         WREG32(CB_PERF_CTR3_SEL_1, 0);
1235
1236         tmp = RREG32(HDP_MISC_CNTL);
1237         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1238         WREG32(HDP_MISC_CNTL, tmp);
1239
1240         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1241         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1242
1243         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1244
1245         udelay(50);
1246
1247         /* set clockgating golden values on TN */
1248         if (rdev->family == CHIP_ARUBA) {
1249                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1250                 tmp &= ~0x00380000;
1251                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1252                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1253                 tmp &= ~0x0e000000;
1254                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1255         }
1256 }
1257
1258 /*
1259  * GART
1260  */
1261 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1262 {
1263         /* flush hdp cache */
1264         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1265
1266         /* bits 0-7 are the VM contexts0-7 */
1267         WREG32(VM_INVALIDATE_REQUEST, 1);
1268 }
1269
1270 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1271 {
1272         int i, r;
1273
1274         if (rdev->gart.robj == NULL) {
1275                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1276                 return -EINVAL;
1277         }
1278         r = radeon_gart_table_vram_pin(rdev);
1279         if (r)
1280                 return r;
1281         /* Setup TLB control */
1282         WREG32(MC_VM_MX_L1_TLB_CNTL,
1283                (0xA << 7) |
1284                ENABLE_L1_TLB |
1285                ENABLE_L1_FRAGMENT_PROCESSING |
1286                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1287                ENABLE_ADVANCED_DRIVER_MODEL |
1288                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1289         /* Setup L2 cache */
1290         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1291                ENABLE_L2_FRAGMENT_PROCESSING |
1292                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1293                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1294                EFFECTIVE_L2_QUEUE_SIZE(7) |
1295                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1296         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1297         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1298                BANK_SELECT(6) |
1299                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1300         /* setup context0 */
1301         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1302         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1303         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1304         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1305                         (u32)(rdev->dummy_page.addr >> 12));
1306         WREG32(VM_CONTEXT0_CNTL2, 0);
1307         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1308                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1309
1310         WREG32(0x15D4, 0);
1311         WREG32(0x15D8, 0);
1312         WREG32(0x15DC, 0);
1313
1314         /* empty context1-7 */
1315         /* Assign the pt base to something valid for now; the pts used for
1316          * the VMs are determined by the application and setup and assigned
1317          * on the fly in the vm part of radeon_gart.c
1318          */
1319         for (i = 1; i < 8; i++) {
1320                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1321                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1322                         rdev->vm_manager.max_pfn - 1);
1323                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1324                        rdev->vm_manager.saved_table_addr[i]);
1325         }
1326
1327         /* enable context1-7 */
1328         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1329                (u32)(rdev->dummy_page.addr >> 12));
1330         WREG32(VM_CONTEXT1_CNTL2, 4);
1331         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1332                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1333                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1334                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1335                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1336                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1337                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1338                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1339                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1340                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1341                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1342                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1343                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1344                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1345
1346         cayman_pcie_gart_tlb_flush(rdev);
1347         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1348                  (unsigned)(rdev->mc.gtt_size >> 20),
1349                  (unsigned long long)rdev->gart.table_addr);
1350         rdev->gart.ready = true;
1351         return 0;
1352 }
1353
1354 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1355 {
1356         unsigned i;
1357
1358         for (i = 1; i < 8; ++i) {
1359                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1360                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1361         }
1362
1363         /* Disable all tables */
1364         WREG32(VM_CONTEXT0_CNTL, 0);
1365         WREG32(VM_CONTEXT1_CNTL, 0);
1366         /* Setup TLB control */
1367         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1368                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1369                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1370         /* Setup L2 cache */
1371         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1372                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1373                EFFECTIVE_L2_QUEUE_SIZE(7) |
1374                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1375         WREG32(VM_L2_CNTL2, 0);
1376         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1377                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1378         radeon_gart_table_vram_unpin(rdev);
1379 }
1380
1381 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1382 {
1383         cayman_pcie_gart_disable(rdev);
1384         radeon_gart_table_vram_free(rdev);
1385         radeon_gart_fini(rdev);
1386 }
1387
1388 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1389                               int ring, u32 cp_int_cntl)
1390 {
1391         WREG32(SRBM_GFX_CNTL, RINGID(ring));
1392         WREG32(CP_INT_CNTL, cp_int_cntl);
1393 }
1394
1395 /*
1396  * CP.
1397  */
1398 void cayman_fence_ring_emit(struct radeon_device *rdev,
1399                             struct radeon_fence *fence)
1400 {
1401         struct radeon_ring *ring = &rdev->ring[fence->ring];
1402         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1403         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1404                 PACKET3_SH_ACTION_ENA;
1405
1406         /* flush read cache over gart for this vmid */
1407         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1408         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1409         radeon_ring_write(ring, 0xFFFFFFFF);
1410         radeon_ring_write(ring, 0);
1411         radeon_ring_write(ring, 10); /* poll interval */
1412         /* EVENT_WRITE_EOP - flush caches, send int */
1413         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1414         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1415         radeon_ring_write(ring, lower_32_bits(addr));
1416         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1417         radeon_ring_write(ring, fence->seq);
1418         radeon_ring_write(ring, 0);
1419 }
1420
1421 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1422 {
1423         struct radeon_ring *ring = &rdev->ring[ib->ring];
1424         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1425         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1426                 PACKET3_SH_ACTION_ENA;
1427
1428         /* set to DX10/11 mode */
1429         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1430         radeon_ring_write(ring, 1);
1431
1432         if (ring->rptr_save_reg) {
1433                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1434                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1435                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1436                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1437                 radeon_ring_write(ring, next_rptr);
1438         }
1439
1440         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1441         radeon_ring_write(ring,
1442 #ifdef __BIG_ENDIAN
1443                           (2 << 0) |
1444 #endif
1445                           (ib->gpu_addr & 0xFFFFFFFC));
1446         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1447         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1448
1449         /* flush read cache over gart for this vmid */
1450         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1451         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1452         radeon_ring_write(ring, 0xFFFFFFFF);
1453         radeon_ring_write(ring, 0);
1454         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1455 }
1456
1457 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1458 {
1459         if (enable)
1460                 WREG32(CP_ME_CNTL, 0);
1461         else {
1462                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1463                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1464                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1465                 WREG32(SCRATCH_UMSK, 0);
1466                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1467         }
1468 }
1469
1470 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1471                         struct radeon_ring *ring)
1472 {
1473         u32 rptr;
1474
1475         if (rdev->wb.enabled)
1476                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1477         else {
1478                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1479                         rptr = RREG32(CP_RB0_RPTR);
1480                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1481                         rptr = RREG32(CP_RB1_RPTR);
1482                 else
1483                         rptr = RREG32(CP_RB2_RPTR);
1484         }
1485
1486         return rptr;
1487 }
1488
1489 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1490                         struct radeon_ring *ring)
1491 {
1492         u32 wptr;
1493
1494         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1495                 wptr = RREG32(CP_RB0_WPTR);
1496         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1497                 wptr = RREG32(CP_RB1_WPTR);
1498         else
1499                 wptr = RREG32(CP_RB2_WPTR);
1500
1501         return wptr;
1502 }
1503
1504 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1505                          struct radeon_ring *ring)
1506 {
1507         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1508                 WREG32(CP_RB0_WPTR, ring->wptr);
1509                 (void)RREG32(CP_RB0_WPTR);
1510         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1511                 WREG32(CP_RB1_WPTR, ring->wptr);
1512                 (void)RREG32(CP_RB1_WPTR);
1513         } else {
1514                 WREG32(CP_RB2_WPTR, ring->wptr);
1515                 (void)RREG32(CP_RB2_WPTR);
1516         }
1517 }
1518
1519 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1520 {
1521         const __be32 *fw_data;
1522         int i;
1523
1524         if (!rdev->me_fw || !rdev->pfp_fw)
1525                 return -EINVAL;
1526
1527         cayman_cp_enable(rdev, false);
1528
1529         fw_data = (const __be32 *)rdev->pfp_fw->data;
1530         WREG32(CP_PFP_UCODE_ADDR, 0);
1531         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1532                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1533         WREG32(CP_PFP_UCODE_ADDR, 0);
1534
1535         fw_data = (const __be32 *)rdev->me_fw->data;
1536         WREG32(CP_ME_RAM_WADDR, 0);
1537         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1538                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1539
1540         WREG32(CP_PFP_UCODE_ADDR, 0);
1541         WREG32(CP_ME_RAM_WADDR, 0);
1542         WREG32(CP_ME_RAM_RADDR, 0);
1543         return 0;
1544 }
1545
1546 static int cayman_cp_start(struct radeon_device *rdev)
1547 {
1548         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1549         int r, i;
1550
1551         r = radeon_ring_lock(rdev, ring, 7);
1552         if (r) {
1553                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1554                 return r;
1555         }
1556         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1557         radeon_ring_write(ring, 0x1);
1558         radeon_ring_write(ring, 0x0);
1559         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1560         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1561         radeon_ring_write(ring, 0);
1562         radeon_ring_write(ring, 0);
1563         radeon_ring_unlock_commit(rdev, ring, false);
1564
1565         cayman_cp_enable(rdev, true);
1566
1567         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1568         if (r) {
1569                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1570                 return r;
1571         }
1572
1573         /* setup clear context state */
1574         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1575         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1576
1577         for (i = 0; i < cayman_default_size; i++)
1578                 radeon_ring_write(ring, cayman_default_state[i]);
1579
1580         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1581         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1582
1583         /* set clear context state */
1584         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1585         radeon_ring_write(ring, 0);
1586
1587         /* SQ_VTX_BASE_VTX_LOC */
1588         radeon_ring_write(ring, 0xc0026f00);
1589         radeon_ring_write(ring, 0x00000000);
1590         radeon_ring_write(ring, 0x00000000);
1591         radeon_ring_write(ring, 0x00000000);
1592
1593         /* Clear consts */
1594         radeon_ring_write(ring, 0xc0036f00);
1595         radeon_ring_write(ring, 0x00000bc4);
1596         radeon_ring_write(ring, 0xffffffff);
1597         radeon_ring_write(ring, 0xffffffff);
1598         radeon_ring_write(ring, 0xffffffff);
1599
1600         radeon_ring_write(ring, 0xc0026900);
1601         radeon_ring_write(ring, 0x00000316);
1602         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1603         radeon_ring_write(ring, 0x00000010); /*  */
1604
1605         radeon_ring_unlock_commit(rdev, ring, false);
1606
1607         /* XXX init other rings */
1608
1609         return 0;
1610 }
1611
1612 static void cayman_cp_fini(struct radeon_device *rdev)
1613 {
1614         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1615         cayman_cp_enable(rdev, false);
1616         radeon_ring_fini(rdev, ring);
1617         radeon_scratch_free(rdev, ring->rptr_save_reg);
1618 }
1619
1620 static int cayman_cp_resume(struct radeon_device *rdev)
1621 {
1622         static const int ridx[] = {
1623                 RADEON_RING_TYPE_GFX_INDEX,
1624                 CAYMAN_RING_TYPE_CP1_INDEX,
1625                 CAYMAN_RING_TYPE_CP2_INDEX
1626         };
1627         static const unsigned cp_rb_cntl[] = {
1628                 CP_RB0_CNTL,
1629                 CP_RB1_CNTL,
1630                 CP_RB2_CNTL,
1631         };
1632         static const unsigned cp_rb_rptr_addr[] = {
1633                 CP_RB0_RPTR_ADDR,
1634                 CP_RB1_RPTR_ADDR,
1635                 CP_RB2_RPTR_ADDR
1636         };
1637         static const unsigned cp_rb_rptr_addr_hi[] = {
1638                 CP_RB0_RPTR_ADDR_HI,
1639                 CP_RB1_RPTR_ADDR_HI,
1640                 CP_RB2_RPTR_ADDR_HI
1641         };
1642         static const unsigned cp_rb_base[] = {
1643                 CP_RB0_BASE,
1644                 CP_RB1_BASE,
1645                 CP_RB2_BASE
1646         };
1647         static const unsigned cp_rb_rptr[] = {
1648                 CP_RB0_RPTR,
1649                 CP_RB1_RPTR,
1650                 CP_RB2_RPTR
1651         };
1652         static const unsigned cp_rb_wptr[] = {
1653                 CP_RB0_WPTR,
1654                 CP_RB1_WPTR,
1655                 CP_RB2_WPTR
1656         };
1657         struct radeon_ring *ring;
1658         int i, r;
1659
1660         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1661         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1662                                  SOFT_RESET_PA |
1663                                  SOFT_RESET_SH |
1664                                  SOFT_RESET_VGT |
1665                                  SOFT_RESET_SPI |
1666                                  SOFT_RESET_SX));
1667         RREG32(GRBM_SOFT_RESET);
1668         mdelay(15);
1669         WREG32(GRBM_SOFT_RESET, 0);
1670         RREG32(GRBM_SOFT_RESET);
1671
1672         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1673         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1674
1675         /* Set the write pointer delay */
1676         WREG32(CP_RB_WPTR_DELAY, 0);
1677
1678         WREG32(CP_DEBUG, (1 << 27));
1679
1680         /* set the wb address whether it's enabled or not */
1681         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1682         WREG32(SCRATCH_UMSK, 0xff);
1683
1684         for (i = 0; i < 3; ++i) {
1685                 uint32_t rb_cntl;
1686                 uint64_t addr;
1687
1688                 /* Set ring buffer size */
1689                 ring = &rdev->ring[ridx[i]];
1690                 rb_cntl = order_base_2(ring->ring_size / 8);
1691                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1692 #ifdef __BIG_ENDIAN
1693                 rb_cntl |= BUF_SWAP_32BIT;
1694 #endif
1695                 WREG32(cp_rb_cntl[i], rb_cntl);
1696
1697                 /* set the wb address whether it's enabled or not */
1698                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1699                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1700                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1701         }
1702
1703         /* set the rb base addr, this causes an internal reset of ALL rings */
1704         for (i = 0; i < 3; ++i) {
1705                 ring = &rdev->ring[ridx[i]];
1706                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1707         }
1708
1709         for (i = 0; i < 3; ++i) {
1710                 /* Initialize the ring buffer's read and write pointers */
1711                 ring = &rdev->ring[ridx[i]];
1712                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1713
1714                 ring->wptr = 0;
1715                 WREG32(cp_rb_rptr[i], 0);
1716                 WREG32(cp_rb_wptr[i], ring->wptr);
1717
1718                 mdelay(1);
1719                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1720         }
1721
1722         /* start the rings */
1723         cayman_cp_start(rdev);
1724         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1725         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1726         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1727         /* this only test cp0 */
1728         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1729         if (r) {
1730                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1731                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1732                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1733                 return r;
1734         }
1735
1736         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1737                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1738
1739         return 0;
1740 }
1741
1742 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1743 {
1744         u32 reset_mask = 0;
1745         u32 tmp;
1746
1747         /* GRBM_STATUS */
1748         tmp = RREG32(GRBM_STATUS);
1749         if (tmp & (PA_BUSY | SC_BUSY |
1750                    SH_BUSY | SX_BUSY |
1751                    TA_BUSY | VGT_BUSY |
1752                    DB_BUSY | CB_BUSY |
1753                    GDS_BUSY | SPI_BUSY |
1754                    IA_BUSY | IA_BUSY_NO_DMA))
1755                 reset_mask |= RADEON_RESET_GFX;
1756
1757         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1758                    CP_BUSY | CP_COHERENCY_BUSY))
1759                 reset_mask |= RADEON_RESET_CP;
1760
1761         if (tmp & GRBM_EE_BUSY)
1762                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1763
1764         /* DMA_STATUS_REG 0 */
1765         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1766         if (!(tmp & DMA_IDLE))
1767                 reset_mask |= RADEON_RESET_DMA;
1768
1769         /* DMA_STATUS_REG 1 */
1770         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1771         if (!(tmp & DMA_IDLE))
1772                 reset_mask |= RADEON_RESET_DMA1;
1773
1774         /* SRBM_STATUS2 */
1775         tmp = RREG32(SRBM_STATUS2);
1776         if (tmp & DMA_BUSY)
1777                 reset_mask |= RADEON_RESET_DMA;
1778
1779         if (tmp & DMA1_BUSY)
1780                 reset_mask |= RADEON_RESET_DMA1;
1781
1782         /* SRBM_STATUS */
1783         tmp = RREG32(SRBM_STATUS);
1784         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1785                 reset_mask |= RADEON_RESET_RLC;
1786
1787         if (tmp & IH_BUSY)
1788                 reset_mask |= RADEON_RESET_IH;
1789
1790         if (tmp & SEM_BUSY)
1791                 reset_mask |= RADEON_RESET_SEM;
1792
1793         if (tmp & GRBM_RQ_PENDING)
1794                 reset_mask |= RADEON_RESET_GRBM;
1795
1796         if (tmp & VMC_BUSY)
1797                 reset_mask |= RADEON_RESET_VMC;
1798
1799         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1800                    MCC_BUSY | MCD_BUSY))
1801                 reset_mask |= RADEON_RESET_MC;
1802
1803         if (evergreen_is_display_hung(rdev))
1804                 reset_mask |= RADEON_RESET_DISPLAY;
1805
1806         /* VM_L2_STATUS */
1807         tmp = RREG32(VM_L2_STATUS);
1808         if (tmp & L2_BUSY)
1809                 reset_mask |= RADEON_RESET_VMC;
1810
1811         /* Skip MC reset as it's mostly likely not hung, just busy */
1812         if (reset_mask & RADEON_RESET_MC) {
1813                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1814                 reset_mask &= ~RADEON_RESET_MC;
1815         }
1816
1817         return reset_mask;
1818 }
1819
1820 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1821 {
1822         struct evergreen_mc_save save;
1823         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1824         u32 tmp;
1825
1826         if (reset_mask == 0)
1827                 return;
1828
1829         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1830
1831         evergreen_print_gpu_status_regs(rdev);
1832         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1833                  RREG32(0x14F8));
1834         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1835                  RREG32(0x14D8));
1836         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1837                  RREG32(0x14FC));
1838         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1839                  RREG32(0x14DC));
1840
1841         /* Disable CP parsing/prefetching */
1842         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1843
1844         if (reset_mask & RADEON_RESET_DMA) {
1845                 /* dma0 */
1846                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1847                 tmp &= ~DMA_RB_ENABLE;
1848                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1849         }
1850
1851         if (reset_mask & RADEON_RESET_DMA1) {
1852                 /* dma1 */
1853                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1854                 tmp &= ~DMA_RB_ENABLE;
1855                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1856         }
1857
1858         udelay(50);
1859
1860         evergreen_mc_stop(rdev, &save);
1861         if (evergreen_mc_wait_for_idle(rdev)) {
1862                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1863         }
1864
1865         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1866                 grbm_soft_reset = SOFT_RESET_CB |
1867                         SOFT_RESET_DB |
1868                         SOFT_RESET_GDS |
1869                         SOFT_RESET_PA |
1870                         SOFT_RESET_SC |
1871                         SOFT_RESET_SPI |
1872                         SOFT_RESET_SH |
1873                         SOFT_RESET_SX |
1874                         SOFT_RESET_TC |
1875                         SOFT_RESET_TA |
1876                         SOFT_RESET_VGT |
1877                         SOFT_RESET_IA;
1878         }
1879
1880         if (reset_mask & RADEON_RESET_CP) {
1881                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1882
1883                 srbm_soft_reset |= SOFT_RESET_GRBM;
1884         }
1885
1886         if (reset_mask & RADEON_RESET_DMA)
1887                 srbm_soft_reset |= SOFT_RESET_DMA;
1888
1889         if (reset_mask & RADEON_RESET_DMA1)
1890                 srbm_soft_reset |= SOFT_RESET_DMA1;
1891
1892         if (reset_mask & RADEON_RESET_DISPLAY)
1893                 srbm_soft_reset |= SOFT_RESET_DC;
1894
1895         if (reset_mask & RADEON_RESET_RLC)
1896                 srbm_soft_reset |= SOFT_RESET_RLC;
1897
1898         if (reset_mask & RADEON_RESET_SEM)
1899                 srbm_soft_reset |= SOFT_RESET_SEM;
1900
1901         if (reset_mask & RADEON_RESET_IH)
1902                 srbm_soft_reset |= SOFT_RESET_IH;
1903
1904         if (reset_mask & RADEON_RESET_GRBM)
1905                 srbm_soft_reset |= SOFT_RESET_GRBM;
1906
1907         if (reset_mask & RADEON_RESET_VMC)
1908                 srbm_soft_reset |= SOFT_RESET_VMC;
1909
1910         if (!(rdev->flags & RADEON_IS_IGP)) {
1911                 if (reset_mask & RADEON_RESET_MC)
1912                         srbm_soft_reset |= SOFT_RESET_MC;
1913         }
1914
1915         if (grbm_soft_reset) {
1916                 tmp = RREG32(GRBM_SOFT_RESET);
1917                 tmp |= grbm_soft_reset;
1918                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1919                 WREG32(GRBM_SOFT_RESET, tmp);
1920                 tmp = RREG32(GRBM_SOFT_RESET);
1921
1922                 udelay(50);
1923
1924                 tmp &= ~grbm_soft_reset;
1925                 WREG32(GRBM_SOFT_RESET, tmp);
1926                 tmp = RREG32(GRBM_SOFT_RESET);
1927         }
1928
1929         if (srbm_soft_reset) {
1930                 tmp = RREG32(SRBM_SOFT_RESET);
1931                 tmp |= srbm_soft_reset;
1932                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1933                 WREG32(SRBM_SOFT_RESET, tmp);
1934                 tmp = RREG32(SRBM_SOFT_RESET);
1935
1936                 udelay(50);
1937
1938                 tmp &= ~srbm_soft_reset;
1939                 WREG32(SRBM_SOFT_RESET, tmp);
1940                 tmp = RREG32(SRBM_SOFT_RESET);
1941         }
1942
1943         /* Wait a little for things to settle down */
1944         udelay(50);
1945
1946         evergreen_mc_resume(rdev, &save);
1947         udelay(50);
1948
1949         evergreen_print_gpu_status_regs(rdev);
1950 }
1951
1952 int cayman_asic_reset(struct radeon_device *rdev, bool hard)
1953 {
1954         u32 reset_mask;
1955
1956         if (hard) {
1957                 evergreen_gpu_pci_config_reset(rdev);
1958                 return 0;
1959         }
1960
1961         reset_mask = cayman_gpu_check_soft_reset(rdev);
1962
1963         if (reset_mask)
1964                 r600_set_bios_scratch_engine_hung(rdev, true);
1965
1966         cayman_gpu_soft_reset(rdev, reset_mask);
1967
1968         reset_mask = cayman_gpu_check_soft_reset(rdev);
1969
1970         if (reset_mask)
1971                 evergreen_gpu_pci_config_reset(rdev);
1972
1973         r600_set_bios_scratch_engine_hung(rdev, false);
1974
1975         return 0;
1976 }
1977
1978 /**
1979  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1980  *
1981  * @rdev: radeon_device pointer
1982  * @ring: radeon_ring structure holding ring information
1983  *
1984  * Check if the GFX engine is locked up.
1985  * Returns true if the engine appears to be locked up, false if not.
1986  */
1987 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1988 {
1989         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1990
1991         if (!(reset_mask & (RADEON_RESET_GFX |
1992                             RADEON_RESET_COMPUTE |
1993                             RADEON_RESET_CP))) {
1994                 radeon_ring_lockup_update(rdev, ring);
1995                 return false;
1996         }
1997         return radeon_ring_test_lockup(rdev, ring);
1998 }
1999
2000 static void cayman_uvd_init(struct radeon_device *rdev)
2001 {
2002         int r;
2003
2004         if (!rdev->has_uvd)
2005                 return;
2006
2007         r = radeon_uvd_init(rdev);
2008         if (r) {
2009                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
2010                 /*
2011                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
2012                  * to early fails uvd_v2_2_resume() and thus nothing happens
2013                  * there. So it is pointless to try to go through that code
2014                  * hence why we disable uvd here.
2015                  */
2016                 rdev->has_uvd = 0;
2017                 return;
2018         }
2019         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
2020         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
2021 }
2022
2023 static void cayman_uvd_start(struct radeon_device *rdev)
2024 {
2025         int r;
2026
2027         if (!rdev->has_uvd)
2028                 return;
2029
2030         r = uvd_v2_2_resume(rdev);
2031         if (r) {
2032                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
2033                 goto error;
2034         }
2035         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
2036         if (r) {
2037                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
2038                 goto error;
2039         }
2040         return;
2041
2042 error:
2043         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2044 }
2045
2046 static void cayman_uvd_resume(struct radeon_device *rdev)
2047 {
2048         struct radeon_ring *ring;
2049         int r;
2050
2051         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
2052                 return;
2053
2054         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2055         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
2056         if (r) {
2057                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
2058                 return;
2059         }
2060         r = uvd_v1_0_init(rdev);
2061         if (r) {
2062                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
2063                 return;
2064         }
2065 }
2066
2067 static void cayman_vce_init(struct radeon_device *rdev)
2068 {
2069         int r;
2070
2071         /* Only set for CHIP_ARUBA */
2072         if (!rdev->has_vce)
2073                 return;
2074
2075         r = radeon_vce_init(rdev);
2076         if (r) {
2077                 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
2078                 /*
2079                  * At this point rdev->vce.vcpu_bo is NULL which trickles down
2080                  * to early fails cayman_vce_start() and thus nothing happens
2081                  * there. So it is pointless to try to go through that code
2082                  * hence why we disable vce here.
2083                  */
2084                 rdev->has_vce = 0;
2085                 return;
2086         }
2087         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
2088         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
2089         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
2090         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
2091 }
2092
2093 static void cayman_vce_start(struct radeon_device *rdev)
2094 {
2095         int r;
2096
2097         if (!rdev->has_vce)
2098                 return;
2099
2100         r = radeon_vce_resume(rdev);
2101         if (r) {
2102                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2103                 goto error;
2104         }
2105         r = vce_v1_0_resume(rdev);
2106         if (r) {
2107                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2108                 goto error;
2109         }
2110         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
2111         if (r) {
2112                 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
2113                 goto error;
2114         }
2115         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
2116         if (r) {
2117                 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
2118                 goto error;
2119         }
2120         return;
2121
2122 error:
2123         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2124         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2125 }
2126
2127 static void cayman_vce_resume(struct radeon_device *rdev)
2128 {
2129         struct radeon_ring *ring;
2130         int r;
2131
2132         if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
2133                 return;
2134
2135         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2136         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2137         if (r) {
2138                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2139                 return;
2140         }
2141         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2142         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2143         if (r) {
2144                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2145                 return;
2146         }
2147         r = vce_v1_0_init(rdev);
2148         if (r) {
2149                 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
2150                 return;
2151         }
2152 }
2153
2154 static int cayman_startup(struct radeon_device *rdev)
2155 {
2156         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2157         int r;
2158
2159         /* enable pcie gen2 link */
2160         evergreen_pcie_gen2_enable(rdev);
2161         /* enable aspm */
2162         evergreen_program_aspm(rdev);
2163
2164         /* scratch needs to be initialized before MC */
2165         r = r600_vram_scratch_init(rdev);
2166         if (r)
2167                 return r;
2168
2169         evergreen_mc_program(rdev);
2170
2171         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2172                 r = ni_mc_load_microcode(rdev);
2173                 if (r) {
2174                         DRM_ERROR("Failed to load MC firmware!\n");
2175                         return r;
2176                 }
2177         }
2178
2179         r = cayman_pcie_gart_enable(rdev);
2180         if (r)
2181                 return r;
2182         cayman_gpu_init(rdev);
2183
2184         /* allocate rlc buffers */
2185         if (rdev->flags & RADEON_IS_IGP) {
2186                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2187                 rdev->rlc.reg_list_size =
2188                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2189                 rdev->rlc.cs_data = cayman_cs_data;
2190                 r = sumo_rlc_init(rdev);
2191                 if (r) {
2192                         DRM_ERROR("Failed to init rlc BOs!\n");
2193                         return r;
2194                 }
2195         }
2196
2197         /* allocate wb buffer */
2198         r = radeon_wb_init(rdev);
2199         if (r)
2200                 return r;
2201
2202         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2203         if (r) {
2204                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2205                 return r;
2206         }
2207
2208         cayman_uvd_start(rdev);
2209         cayman_vce_start(rdev);
2210
2211         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2212         if (r) {
2213                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2214                 return r;
2215         }
2216
2217         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2218         if (r) {
2219                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2220                 return r;
2221         }
2222
2223         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2224         if (r) {
2225                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2226                 return r;
2227         }
2228
2229         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2230         if (r) {
2231                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2232                 return r;
2233         }
2234
2235         /* Enable IRQ */
2236         if (!rdev->irq.installed) {
2237                 r = radeon_irq_kms_init(rdev);
2238                 if (r)
2239                         return r;
2240         }
2241
2242         r = r600_irq_init(rdev);
2243         if (r) {
2244                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2245                 radeon_irq_kms_fini(rdev);
2246                 return r;
2247         }
2248         evergreen_irq_set(rdev);
2249
2250         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2251                              RADEON_CP_PACKET2);
2252         if (r)
2253                 return r;
2254
2255         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2256         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2257                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2258         if (r)
2259                 return r;
2260
2261         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2262         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2263                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2264         if (r)
2265                 return r;
2266
2267         r = cayman_cp_load_microcode(rdev);
2268         if (r)
2269                 return r;
2270         r = cayman_cp_resume(rdev);
2271         if (r)
2272                 return r;
2273
2274         r = cayman_dma_resume(rdev);
2275         if (r)
2276                 return r;
2277
2278         cayman_uvd_resume(rdev);
2279         cayman_vce_resume(rdev);
2280
2281         r = radeon_ib_pool_init(rdev);
2282         if (r) {
2283                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2284                 return r;
2285         }
2286
2287         r = radeon_vm_manager_init(rdev);
2288         if (r) {
2289                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2290                 return r;
2291         }
2292
2293         r = radeon_audio_init(rdev);
2294         if (r)
2295                 return r;
2296
2297         return 0;
2298 }
2299
2300 int cayman_resume(struct radeon_device *rdev)
2301 {
2302         int r;
2303
2304         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2305          * posting will perform necessary task to bring back GPU into good
2306          * shape.
2307          */
2308         /* post card */
2309         atom_asic_init(rdev->mode_info.atom_context);
2310
2311         /* init golden registers */
2312         ni_init_golden_registers(rdev);
2313
2314         if (rdev->pm.pm_method == PM_METHOD_DPM)
2315                 radeon_pm_resume(rdev);
2316
2317         rdev->accel_working = true;
2318         r = cayman_startup(rdev);
2319         if (r) {
2320                 DRM_ERROR("cayman startup failed on resume\n");
2321                 rdev->accel_working = false;
2322                 return r;
2323         }
2324         return r;
2325 }
2326
2327 int cayman_suspend(struct radeon_device *rdev)
2328 {
2329         radeon_pm_suspend(rdev);
2330         radeon_audio_fini(rdev);
2331         radeon_vm_manager_fini(rdev);
2332         cayman_cp_enable(rdev, false);
2333         cayman_dma_stop(rdev);
2334         if (rdev->has_uvd) {
2335                 uvd_v1_0_fini(rdev);
2336                 radeon_uvd_suspend(rdev);
2337         }
2338         evergreen_irq_suspend(rdev);
2339         radeon_wb_disable(rdev);
2340         cayman_pcie_gart_disable(rdev);
2341         return 0;
2342 }
2343
2344 /* Plan is to move initialization in that function and use
2345  * helper function so that radeon_device_init pretty much
2346  * do nothing more than calling asic specific function. This
2347  * should also allow to remove a bunch of callback function
2348  * like vram_info.
2349  */
2350 int cayman_init(struct radeon_device *rdev)
2351 {
2352         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2353         int r;
2354
2355         /* Read BIOS */
2356         if (!radeon_get_bios(rdev)) {
2357                 if (ASIC_IS_AVIVO(rdev))
2358                         return -EINVAL;
2359         }
2360         /* Must be an ATOMBIOS */
2361         if (!rdev->is_atom_bios) {
2362                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2363                 return -EINVAL;
2364         }
2365         r = radeon_atombios_init(rdev);
2366         if (r)
2367                 return r;
2368
2369         /* Post card if necessary */
2370         if (!radeon_card_posted(rdev)) {
2371                 if (!rdev->bios) {
2372                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2373                         return -EINVAL;
2374                 }
2375                 DRM_INFO("GPU not posted. posting now...\n");
2376                 atom_asic_init(rdev->mode_info.atom_context);
2377         }
2378         /* init golden registers */
2379         ni_init_golden_registers(rdev);
2380         /* Initialize scratch registers */
2381         r600_scratch_init(rdev);
2382         /* Initialize surface registers */
2383         radeon_surface_init(rdev);
2384         /* Initialize clocks */
2385         radeon_get_clock_info(rdev->ddev);
2386         /* Fence driver */
2387         r = radeon_fence_driver_init(rdev);
2388         if (r)
2389                 return r;
2390         /* initialize memory controller */
2391         r = evergreen_mc_init(rdev);
2392         if (r)
2393                 return r;
2394         /* Memory manager */
2395         r = radeon_bo_init(rdev);
2396         if (r)
2397                 return r;
2398
2399         if (rdev->flags & RADEON_IS_IGP) {
2400                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2401                         r = ni_init_microcode(rdev);
2402                         if (r) {
2403                                 DRM_ERROR("Failed to load firmware!\n");
2404                                 return r;
2405                         }
2406                 }
2407         } else {
2408                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2409                         r = ni_init_microcode(rdev);
2410                         if (r) {
2411                                 DRM_ERROR("Failed to load firmware!\n");
2412                                 return r;
2413                         }
2414                 }
2415         }
2416
2417         /* Initialize power management */
2418         radeon_pm_init(rdev);
2419
2420         ring->ring_obj = NULL;
2421         r600_ring_init(rdev, ring, 1024 * 1024);
2422
2423         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2424         ring->ring_obj = NULL;
2425         r600_ring_init(rdev, ring, 64 * 1024);
2426
2427         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2428         ring->ring_obj = NULL;
2429         r600_ring_init(rdev, ring, 64 * 1024);
2430
2431         cayman_uvd_init(rdev);
2432         cayman_vce_init(rdev);
2433
2434         rdev->ih.ring_obj = NULL;
2435         r600_ih_ring_init(rdev, 64 * 1024);
2436
2437         r = r600_pcie_gart_init(rdev);
2438         if (r)
2439                 return r;
2440
2441         rdev->accel_working = true;
2442         r = cayman_startup(rdev);
2443         if (r) {
2444                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2445                 cayman_cp_fini(rdev);
2446                 cayman_dma_fini(rdev);
2447                 r600_irq_fini(rdev);
2448                 if (rdev->flags & RADEON_IS_IGP)
2449                         sumo_rlc_fini(rdev);
2450                 radeon_wb_fini(rdev);
2451                 radeon_ib_pool_fini(rdev);
2452                 radeon_vm_manager_fini(rdev);
2453                 radeon_irq_kms_fini(rdev);
2454                 cayman_pcie_gart_fini(rdev);
2455                 rdev->accel_working = false;
2456         }
2457
2458         /* Don't start up if the MC ucode is missing.
2459          * The default clocks and voltages before the MC ucode
2460          * is loaded are not suffient for advanced operations.
2461          *
2462          * We can skip this check for TN, because there is no MC
2463          * ucode.
2464          */
2465         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2466                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2467                 return -EINVAL;
2468         }
2469
2470         return 0;
2471 }
2472
2473 void cayman_fini(struct radeon_device *rdev)
2474 {
2475         radeon_pm_fini(rdev);
2476         cayman_cp_fini(rdev);
2477         cayman_dma_fini(rdev);
2478         r600_irq_fini(rdev);
2479         if (rdev->flags & RADEON_IS_IGP)
2480                 sumo_rlc_fini(rdev);
2481         radeon_wb_fini(rdev);
2482         radeon_vm_manager_fini(rdev);
2483         radeon_ib_pool_fini(rdev);
2484         radeon_irq_kms_fini(rdev);
2485         uvd_v1_0_fini(rdev);
2486         radeon_uvd_fini(rdev);
2487         if (rdev->has_vce)
2488                 radeon_vce_fini(rdev);
2489         cayman_pcie_gart_fini(rdev);
2490         r600_vram_scratch_fini(rdev);
2491         radeon_gem_fini(rdev);
2492         radeon_fence_driver_fini(rdev);
2493         radeon_bo_fini(rdev);
2494         radeon_atombios_fini(rdev);
2495         kfree(rdev->bios);
2496         rdev->bios = NULL;
2497 }
2498
2499 /*
2500  * vm
2501  */
2502 int cayman_vm_init(struct radeon_device *rdev)
2503 {
2504         /* number of VMs */
2505         rdev->vm_manager.nvm = 8;
2506         /* base offset of vram pages */
2507         if (rdev->flags & RADEON_IS_IGP) {
2508                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2509                 tmp <<= 22;
2510                 rdev->vm_manager.vram_base_offset = tmp;
2511         } else
2512                 rdev->vm_manager.vram_base_offset = 0;
2513         return 0;
2514 }
2515
2516 void cayman_vm_fini(struct radeon_device *rdev)
2517 {
2518 }
2519
2520 /**
2521  * cayman_vm_decode_fault - print human readable fault info
2522  *
2523  * @rdev: radeon_device pointer
2524  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2525  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2526  *
2527  * Print human readable fault information (cayman/TN).
2528  */
2529 void cayman_vm_decode_fault(struct radeon_device *rdev,
2530                             u32 status, u32 addr)
2531 {
2532         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2533         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2534         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2535         char *block;
2536
2537         switch (mc_id) {
2538         case 32:
2539         case 16:
2540         case 96:
2541         case 80:
2542         case 160:
2543         case 144:
2544         case 224:
2545         case 208:
2546                 block = "CB";
2547                 break;
2548         case 33:
2549         case 17:
2550         case 97:
2551         case 81:
2552         case 161:
2553         case 145:
2554         case 225:
2555         case 209:
2556                 block = "CB_FMASK";
2557                 break;
2558         case 34:
2559         case 18:
2560         case 98:
2561         case 82:
2562         case 162:
2563         case 146:
2564         case 226:
2565         case 210:
2566                 block = "CB_CMASK";
2567                 break;
2568         case 35:
2569         case 19:
2570         case 99:
2571         case 83:
2572         case 163:
2573         case 147:
2574         case 227:
2575         case 211:
2576                 block = "CB_IMMED";
2577                 break;
2578         case 36:
2579         case 20:
2580         case 100:
2581         case 84:
2582         case 164:
2583         case 148:
2584         case 228:
2585         case 212:
2586                 block = "DB";
2587                 break;
2588         case 37:
2589         case 21:
2590         case 101:
2591         case 85:
2592         case 165:
2593         case 149:
2594         case 229:
2595         case 213:
2596                 block = "DB_HTILE";
2597                 break;
2598         case 38:
2599         case 22:
2600         case 102:
2601         case 86:
2602         case 166:
2603         case 150:
2604         case 230:
2605         case 214:
2606                 block = "SX";
2607                 break;
2608         case 39:
2609         case 23:
2610         case 103:
2611         case 87:
2612         case 167:
2613         case 151:
2614         case 231:
2615         case 215:
2616                 block = "DB_STEN";
2617                 break;
2618         case 40:
2619         case 24:
2620         case 104:
2621         case 88:
2622         case 232:
2623         case 216:
2624         case 168:
2625         case 152:
2626                 block = "TC_TFETCH";
2627                 break;
2628         case 41:
2629         case 25:
2630         case 105:
2631         case 89:
2632         case 233:
2633         case 217:
2634         case 169:
2635         case 153:
2636                 block = "TC_VFETCH";
2637                 break;
2638         case 42:
2639         case 26:
2640         case 106:
2641         case 90:
2642         case 234:
2643         case 218:
2644         case 170:
2645         case 154:
2646                 block = "VC";
2647                 break;
2648         case 112:
2649                 block = "CP";
2650                 break;
2651         case 113:
2652         case 114:
2653                 block = "SH";
2654                 break;
2655         case 115:
2656                 block = "VGT";
2657                 break;
2658         case 178:
2659                 block = "IH";
2660                 break;
2661         case 51:
2662                 block = "RLC";
2663                 break;
2664         case 55:
2665                 block = "DMA";
2666                 break;
2667         case 56:
2668                 block = "HDP";
2669                 break;
2670         default:
2671                 block = "unknown";
2672                 break;
2673         }
2674
2675         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2676                protections, vmid, addr,
2677                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2678                block, mc_id);
2679 }
2680
2681 /**
2682  * cayman_vm_flush - vm flush using the CP
2683  *
2684  * @rdev: radeon_device pointer
2685  *
2686  * Update the page table base and flush the VM TLB
2687  * using the CP (cayman-si).
2688  */
2689 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2690                      unsigned vm_id, uint64_t pd_addr)
2691 {
2692         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2693         radeon_ring_write(ring, pd_addr >> 12);
2694
2695         /* flush hdp cache */
2696         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2697         radeon_ring_write(ring, 0x1);
2698
2699         /* bits 0-7 are the VM contexts0-7 */
2700         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2701         radeon_ring_write(ring, 1 << vm_id);
2702
2703         /* wait for the invalidate to complete */
2704         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2705         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2706                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2707         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2708         radeon_ring_write(ring, 0);
2709         radeon_ring_write(ring, 0); /* ref */
2710         radeon_ring_write(ring, 0); /* mask */
2711         radeon_ring_write(ring, 0x20); /* poll interval */
2712
2713         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2714         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2715         radeon_ring_write(ring, 0x0);
2716 }
2717
2718 int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2719 {
2720         struct atom_clock_dividers dividers;
2721         int r, i;
2722
2723         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2724                                            ecclk, false, &dividers);
2725         if (r)
2726                 return r;
2727
2728         for (i = 0; i < 100; i++) {
2729                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2730                         break;
2731                 mdelay(10);
2732         }
2733         if (i == 100)
2734                 return -ETIMEDOUT;
2735
2736         WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2737
2738         for (i = 0; i < 100; i++) {
2739                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2740                         break;
2741                 mdelay(10);
2742         }
2743         if (i == 100)
2744                 return -ETIMEDOUT;
2745
2746         return 0;
2747 }