drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/kthread.h>
  29 #include <linux/console.h>
  30 #include <linux/slab.h>
  31 #include <linux/debugfs.h>
  32 #include <drm/drmP.h>
  33 #include <drm/drm_crtc_helper.h>
  34 #include <drm/amdgpu_drm.h>
  35 #include <linux/vgaarb.h>
  36 #include <linux/vga_switcheroo.h>
  37 #include <linux/efi.h>
  38 #include "amdgpu.h"
  39 #include "amdgpu_trace.h"
  40 #include "amdgpu_i2c.h"
  41 #include "atom.h"
  42 #include "amdgpu_atombios.h"
  43 #include "amdgpu_atomfirmware.h"
  44 #include "amd_pcie.h"
  45 #ifdef CONFIG_DRM_AMDGPU_SI
  46 #include "si.h"
  47 #endif
  48 #ifdef CONFIG_DRM_AMDGPU_CIK
  49 #include "cik.h"
  50 #endif
  51 #include "vi.h"
  52 #include "soc15.h"
  53 #include "bif/bif_4_1_d.h"
  54 #include <linux/pci.h>
  55 #include <linux/firmware.h>
  56 #include "amdgpu_vf_error.h"
  57
  58 #include "amdgpu_amdkfd.h"
  59 #include "amdgpu_pm.h"
  60
  61 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  62 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  63
  64 #define AMDGPU_RESUME_MS                2000
  65
  66 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
  67 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
  68 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev);
  69 static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev);
  70
  71 static const char *amdgpu_asic_name[] = {
  72         "TAHITI",
  73         "PITCAIRN",
  74         "VERDE",
  75         "OLAND",
  76         "HAINAN",
  77         "BONAIRE",
  78         "KAVERI",
  79         "KABINI",
  80         "HAWAII",
  81         "MULLINS",
  82         "TOPAZ",
  83         "TONGA",
  84         "FIJI",
  85         "CARRIZO",
  86         "STONEY",
  87         "POLARIS10",
  88         "POLARIS11",
  89         "POLARIS12",
  90         "VEGA10",
  91         "RAVEN",
  92         "LAST",
  93 };
  94
  95 bool amdgpu_device_is_px(struct drm_device *dev)
  96 {
  97         struct amdgpu_device *adev = dev->dev_private;
  98
  99         if (adev->flags & AMD_IS_PX)
 100                 return true;
 101         return false;
 102 }
 103
 104 /*
 105  * MMIO register access helper functions.
 106  */
 107 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
 108                         uint32_t acc_flags)
 109 {
 110         uint32_t ret;
 111
 112         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
 113                 return amdgpu_virt_kiq_rreg(adev, reg);
 114
 115         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 116                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 117         else {
 118                 unsigned long flags;
 119
 120                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 121                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 122                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 123                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 124         }
 125         trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
 126         return ret;
 127 }
 128
 129 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 130                     uint32_t acc_flags)
 131 {
 132         trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 133
 134         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
 135                 adev->last_mm_index = v;
 136         }
 137
 138         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
 139                 return amdgpu_virt_kiq_wreg(adev, reg, v);
 140
 141         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 142                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 143         else {
 144                 unsigned long flags;
 145
 146                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 147                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 148                 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 149                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 150         }
 151
 152         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
 153                 udelay(500);
 154         }
 155 }
 156
 157 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
 158 {
 159         if ((reg * 4) < adev->rio_mem_size)
 160                 return ioread32(adev->rio_mem + (reg * 4));
 161         else {
 162                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 163                 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
 164         }
 165 }
 166
 167 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 168 {
 169         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
 170                 adev->last_mm_index = v;
 171         }
 172
 173         if ((reg * 4) < adev->rio_mem_size)
 174                 iowrite32(v, adev->rio_mem + (reg * 4));
 175         else {
 176                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 177                 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
 178         }
 179
 180         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
 181                 udelay(500);
 182         }
 183 }
 184
 185 /**
 186  * amdgpu_mm_rdoorbell - read a doorbell dword
 187  *
 188  * @adev: amdgpu_device pointer
 189  * @index: doorbell index
 190  *
 191  * Returns the value in the doorbell aperture at the
 192  * requested doorbell index (CIK).
 193  */
 194 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 195 {
 196         if (index < adev->doorbell.num_doorbells) {
 197                 return readl(adev->doorbell.ptr + index);
 198         } else {
 199                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 200                 return 0;
 201         }
 202 }
 203
 204 /**
 205  * amdgpu_mm_wdoorbell - write a doorbell dword
 206  *
 207  * @adev: amdgpu_device pointer
 208  * @index: doorbell index
 209  * @v: value to write
 210  *
 211  * Writes @v to the doorbell aperture at the
 212  * requested doorbell index (CIK).
 213  */
 214 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 215 {
 216         if (index < adev->doorbell.num_doorbells) {
 217                 writel(v, adev->doorbell.ptr + index);
 218         } else {
 219                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 220         }
 221 }
 222
 223 /**
 224  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 225  *
 226  * @adev: amdgpu_device pointer
 227  * @index: doorbell index
 228  *
 229  * Returns the value in the doorbell aperture at the
 230  * requested doorbell index (VEGA10+).
 231  */
 232 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 233 {
 234         if (index < adev->doorbell.num_doorbells) {
 235                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 236         } else {
 237                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 238                 return 0;
 239         }
 240 }
 241
 242 /**
 243  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 244  *
 245  * @adev: amdgpu_device pointer
 246  * @index: doorbell index
 247  * @v: value to write
 248  *
 249  * Writes @v to the doorbell aperture at the
 250  * requested doorbell index (VEGA10+).
 251  */
 252 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 253 {
 254         if (index < adev->doorbell.num_doorbells) {
 255                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 256         } else {
 257                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 258         }
 259 }
 260
 261 /**
 262  * amdgpu_invalid_rreg - dummy reg read function
 263  *
 264  * @adev: amdgpu device pointer
 265  * @reg: offset of register
 266  *
 267  * Dummy register read function.  Used for register blocks
 268  * that certain asics don't have (all asics).
 269  * Returns the value in the register.
 270  */
 271 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 272 {
 273         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 274         BUG();
 275         return 0;
 276 }
 277
 278 /**
 279  * amdgpu_invalid_wreg - dummy reg write function
 280  *
 281  * @adev: amdgpu device pointer
 282  * @reg: offset of register
 283  * @v: value to write to the register
 284  *
 285  * Dummy register read function.  Used for register blocks
 286  * that certain asics don't have (all asics).
 287  */
 288 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 289 {
 290         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 291                   reg, v);
 292         BUG();
 293 }
 294
 295 /**
 296  * amdgpu_block_invalid_rreg - dummy reg read function
 297  *
 298  * @adev: amdgpu device pointer
 299  * @block: offset of instance
 300  * @reg: offset of register
 301  *
 302  * Dummy register read function.  Used for register blocks
 303  * that certain asics don't have (all asics).
 304  * Returns the value in the register.
 305  */
 306 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 307                                           uint32_t block, uint32_t reg)
 308 {
 309         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 310                   reg, block);
 311         BUG();
 312         return 0;
 313 }
 314
 315 /**
 316  * amdgpu_block_invalid_wreg - dummy reg write function
 317  *
 318  * @adev: amdgpu device pointer
 319  * @block: offset of instance
 320  * @reg: offset of register
 321  * @v: value to write to the register
 322  *
 323  * Dummy register read function.  Used for register blocks
 324  * that certain asics don't have (all asics).
 325  */
 326 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 327                                       uint32_t block,
 328                                       uint32_t reg, uint32_t v)
 329 {
 330         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 331                   reg, block, v);
 332         BUG();
 333 }
 334
 335 static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
 336 {
 337         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 338                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 339                                        &adev->vram_scratch.robj,
 340                                        &adev->vram_scratch.gpu_addr,
 341                                        (void **)&adev->vram_scratch.ptr);
 342 }
 343
 344 static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
 345 {
 346         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 347 }
 348
 349 /**
 350  * amdgpu_program_register_sequence - program an array of registers.
 351  *
 352  * @adev: amdgpu_device pointer
 353  * @registers: pointer to the register array
 354  * @array_size: size of the register array
 355  *
 356  * Programs an array or registers with and and or masks.
 357  * This is a helper for setting golden registers.
 358  */
 359 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
 360                                       const u32 *registers,
 361                                       const u32 array_size)
 362 {
 363         u32 tmp, reg, and_mask, or_mask;
 364         int i;
 365
 366         if (array_size % 3)
 367                 return;
 368
 369         for (i = 0; i < array_size; i +=3) {
 370                 reg = registers[i + 0];
 371                 and_mask = registers[i + 1];
 372                 or_mask = registers[i + 2];
 373
 374                 if (and_mask == 0xffffffff) {
 375                         tmp = or_mask;
 376                 } else {
 377                         tmp = RREG32(reg);
 378                         tmp &= ~and_mask;
 379                         tmp |= or_mask;
 380                 }
 381                 WREG32(reg, tmp);
 382         }
 383 }
 384
 385 void amdgpu_pci_config_reset(struct amdgpu_device *adev)
 386 {
 387         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 388 }
 389
 390 /*
 391  * GPU doorbell aperture helpers function.
 392  */
 393 /**
 394  * amdgpu_doorbell_init - Init doorbell driver information.
 395  *
 396  * @adev: amdgpu_device pointer
 397  *
 398  * Init doorbell driver information (CIK)
 399  * Returns 0 on success, error on failure.
 400  */
 401 static int amdgpu_doorbell_init(struct amdgpu_device *adev)
 402 {
 403         /* No doorbell on SI hardware generation */
 404         if (adev->asic_type < CHIP_BONAIRE) {
 405                 adev->doorbell.base = 0;
 406                 adev->doorbell.size = 0;
 407                 adev->doorbell.num_doorbells = 0;
 408                 adev->doorbell.ptr = NULL;
 409                 return 0;
 410         }
 411
 412         /* doorbell bar mapping */
 413         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 414         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 415
 416         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 417                                              AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
 418         if (adev->doorbell.num_doorbells == 0)
 419                 return -EINVAL;
 420
 421         adev->doorbell.ptr = ioremap(adev->doorbell.base,
 422                                      adev->doorbell.num_doorbells *
 423                                      sizeof(u32));
 424         if (adev->doorbell.ptr == NULL)
 425                 return -ENOMEM;
 426
 427         return 0;
 428 }
 429
 430 /**
 431  * amdgpu_doorbell_fini - Tear down doorbell driver information.
 432  *
 433  * @adev: amdgpu_device pointer
 434  *
 435  * Tear down doorbell driver information (CIK)
 436  */
 437 static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
 438 {
 439         iounmap(adev->doorbell.ptr);
 440         adev->doorbell.ptr = NULL;
 441 }
 442
 443 /**
 444  * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
 445  *                                setup amdkfd
 446  *
 447  * @adev: amdgpu_device pointer
 448  * @aperture_base: output returning doorbell aperture base physical address
 449  * @aperture_size: output returning doorbell aperture size in bytes
 450  * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
 451  *
 452  * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
 453  * takes doorbells required for its own rings and reports the setup to amdkfd.
 454  * amdgpu reserved doorbells are at the start of the doorbell aperture.
 455  */
 456 void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 457                                 phys_addr_t *aperture_base,
 458                                 size_t *aperture_size,
 459                                 size_t *start_offset)
 460 {
 461         /*
 462          * The first num_doorbells are used by amdgpu.
 463          * amdkfd takes whatever's left in the aperture.
 464          */
 465         if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
 466                 *aperture_base = adev->doorbell.base;
 467                 *aperture_size = adev->doorbell.size;
 468                 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
 469         } else {
 470                 *aperture_base = 0;
 471                 *aperture_size = 0;
 472                 *start_offset = 0;
 473         }
 474 }
 475
 476 /*
 477  * amdgpu_wb_*()
 478  * Writeback is the method by which the GPU updates special pages in memory
 479  * with the status of certain GPU events (fences, ring pointers,etc.).
 480  */
 481
 482 /**
 483  * amdgpu_wb_fini - Disable Writeback and free memory
 484  *
 485  * @adev: amdgpu_device pointer
 486  *
 487  * Disables Writeback and frees the Writeback memory (all asics).
 488  * Used at driver shutdown.
 489  */
 490 static void amdgpu_wb_fini(struct amdgpu_device *adev)
 491 {
 492         if (adev->wb.wb_obj) {
 493                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
 494                                       &adev->wb.gpu_addr,
 495                                       (void **)&adev->wb.wb);
 496                 adev->wb.wb_obj = NULL;
 497         }
 498 }
 499
 500 /**
 501  * amdgpu_wb_init- Init Writeback driver info and allocate memory
 502  *
 503  * @adev: amdgpu_device pointer
 504  *
 505  * Initializes writeback and allocates writeback memory (all asics).
 506  * Used at driver startup.
 507  * Returns 0 on success or an -error on failure.
 508  */
 509 static int amdgpu_wb_init(struct amdgpu_device *adev)
 510 {
 511         int r;
 512
 513         if (adev->wb.wb_obj == NULL) {
 514                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
 515                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
 516                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 517                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
 518                                             (void **)&adev->wb.wb);
 519                 if (r) {
 520                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
 521                         return r;
 522                 }
 523
 524                 adev->wb.num_wb = AMDGPU_MAX_WB;
 525                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
 526
 527                 /* clear wb memory */
 528                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t));
 529         }
 530
 531         return 0;
 532 }
 533
 534 /**
 535  * amdgpu_wb_get - Allocate a wb entry
 536  *
 537  * @adev: amdgpu_device pointer
 538  * @wb: wb index
 539  *
 540  * Allocate a wb slot for use by the driver (all asics).
 541  * Returns 0 on success or -EINVAL on failure.
 542  */
 543 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
 544 {
 545         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 546
 547         if (offset < adev->wb.num_wb) {
 548                 __set_bit(offset, adev->wb.used);
 549                 *wb = offset << 3; /* convert to dw offset */
 550                 return 0;
 551         } else {
 552                 return -EINVAL;
 553         }
 554 }
 555
 556 /**
 557  * amdgpu_wb_free - Free a wb entry
 558  *
 559  * @adev: amdgpu_device pointer
 560  * @wb: wb index
 561  *
 562  * Free a wb slot allocated for use by the driver (all asics)
 563  */
 564 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
 565 {
 566         if (wb < adev->wb.num_wb)
 567                 __clear_bit(wb >> 3, adev->wb.used);
 568 }
 569
 570 /**
 571  * amdgpu_vram_location - try to find VRAM location
 572  * @adev: amdgpu device structure holding all necessary informations
 573  * @mc: memory controller structure holding memory informations
 574  * @base: base address at which to put VRAM
 575  *
 576  * Function will try to place VRAM at base address provided
 577  * as parameter (which is so far either PCI aperture address or
 578  * for IGP TOM base address).
 579  *
 580  * If there is not enough space to fit the unvisible VRAM in the 32bits
 581  * address space then we limit the VRAM size to the aperture.
 582  *
 583  * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
 584  * this shouldn't be a problem as we are using the PCI aperture as a reference.
 585  * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
 586  * not IGP.
 587  *
 588  * Note: we use mc_vram_size as on some board we need to program the mc to
 589  * cover the whole aperture even if VRAM size is inferior to aperture size
 590  * Novell bug 204882 + along with lots of ubuntu ones
 591  *
 592  * Note: when limiting vram it's safe to overwritte real_vram_size because
 593  * we are not in case where real_vram_size is inferior to mc_vram_size (ie
 594  * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
 595  * ones)
 596  *
 597  * Note: IGP TOM addr should be the same as the aperture addr, we don't
 598  * explicitly check for that though.
 599  *
 600  * FIXME: when reducing VRAM size align new size on power of 2.
 601  */
 602 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
 603 {
 604         uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
 605
 606         mc->vram_start = base;
 607         if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
 608                 dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
 609                 mc->real_vram_size = mc->aper_size;
 610                 mc->mc_vram_size = mc->aper_size;
 611         }
 612         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 613         if (limit && limit < mc->real_vram_size)
 614                 mc->real_vram_size = limit;
 615         dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 616                         mc->mc_vram_size >> 20, mc->vram_start,
 617                         mc->vram_end, mc->real_vram_size >> 20);
 618 }
 619
 620 /**
 621  * amdgpu_gart_location - try to find GTT location
 622  * @adev: amdgpu device structure holding all necessary informations
 623  * @mc: memory controller structure holding memory informations
 624  *
 625  * Function will place try to place GTT before or after VRAM.
 626  *
 627  * If GTT size is bigger than space left then we ajust GTT size.
 628  * Thus function will never fails.
 629  *
 630  * FIXME: when reducing GTT size align new size on power of 2.
 631  */
 632 void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
 633 {
 634         u64 size_af, size_bf;
 635
 636         size_af = adev->mc.mc_mask - mc->vram_end;
 637         size_bf = mc->vram_start;
 638         if (size_bf > size_af) {
 639                 if (mc->gart_size > size_bf) {
 640                         dev_warn(adev->dev, "limiting GTT\n");
 641                         mc->gart_size = size_bf;
 642                 }
 643                 mc->gart_start = 0;
 644         } else {
 645                 if (mc->gart_size > size_af) {
 646                         dev_warn(adev->dev, "limiting GTT\n");
 647                         mc->gart_size = size_af;
 648                 }
 649                 mc->gart_start = mc->vram_end + 1;
 650         }
 651         mc->gart_end = mc->gart_start + mc->gart_size - 1;
 652         dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
 653                         mc->gart_size >> 20, mc->gart_start, mc->gart_end);
 654 }
 655
 656 /*
 657  * Firmware Reservation functions
 658  */
 659 /**
 660  * amdgpu_fw_reserve_vram_fini - free fw reserved vram
 661  *
 662  * @adev: amdgpu_device pointer
 663  *
 664  * free fw reserved vram if it has been reserved.
 665  */
 666 void amdgpu_fw_reserve_vram_fini(struct amdgpu_device *adev)
 667 {
 668         amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
 669                 NULL, &adev->fw_vram_usage.va);
 670 }
 671
 672 /**
 673  * amdgpu_fw_reserve_vram_init - create bo vram reservation from fw
 674  *
 675  * @adev: amdgpu_device pointer
 676  *
 677  * create bo vram reservation from fw.
 678  */
 679 int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
 680 {
 681         int r = 0;
 682         u64 gpu_addr;
 683         u64 vram_size = adev->mc.visible_vram_size;
 684
 685         adev->fw_vram_usage.va = NULL;
 686         adev->fw_vram_usage.reserved_bo = NULL;
 687
 688         if (adev->fw_vram_usage.size > 0 &&
 689                 adev->fw_vram_usage.size <= vram_size) {
 690
 691                 r = amdgpu_bo_create(adev, adev->fw_vram_usage.size,
 692                         PAGE_SIZE, true, 0,
 693                         AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 694                         AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, NULL, NULL, 0,
 695                         &adev->fw_vram_usage.reserved_bo);
 696                 if (r)
 697                         goto error_create;
 698
 699                 r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
 700                 if (r)
 701                         goto error_reserve;
 702                 r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
 703                         AMDGPU_GEM_DOMAIN_VRAM,
 704                         adev->fw_vram_usage.start_offset,
 705                         (adev->fw_vram_usage.start_offset +
 706                         adev->fw_vram_usage.size), &gpu_addr);
 707                 if (r)
 708                         goto error_pin;
 709                 r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
 710                         &adev->fw_vram_usage.va);
 711                 if (r)
 712                         goto error_kmap;
 713
 714                 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
 715         }
 716         return r;
 717
 718 error_kmap:
 719         amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
 720 error_pin:
 721         amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
 722 error_reserve:
 723         amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
 724 error_create:
 725         adev->fw_vram_usage.va = NULL;
 726         adev->fw_vram_usage.reserved_bo = NULL;
 727         return r;
 728 }
 729
 730
 731 /*
 732  * GPU helpers function.
 733  */
 734 /**
 735  * amdgpu_need_post - check if the hw need post or not
 736  *
 737  * @adev: amdgpu_device pointer
 738  *
 739  * Check if the asic has been initialized (all asics) at driver startup
 740  * or post is needed if  hw reset is performed.
 741  * Returns true if need or false if not.
 742  */
 743 bool amdgpu_need_post(struct amdgpu_device *adev)
 744 {
 745         uint32_t reg;
 746
 747         if (amdgpu_sriov_vf(adev))
 748                 return false;
 749
 750         if (amdgpu_passthrough(adev)) {
 751                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
 752                  * some old smc fw still need driver do vPost otherwise gpu hang, while
 753                  * those smc fw version above 22.15 doesn't have this flaw, so we force
 754                  * vpost executed for smc version below 22.15
 755                  */
 756                 if (adev->asic_type == CHIP_FIJI) {
 757                         int err;
 758                         uint32_t fw_ver;
 759                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
 760                         /* force vPost if error occured */
 761                         if (err)
 762                                 return true;
 763
 764                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
 765                         if (fw_ver < 0x00160e00)
 766                                 return true;
 767                 }
 768         }
 769
 770         if (adev->has_hw_reset) {
 771                 adev->has_hw_reset = false;
 772                 return true;
 773         }
 774
 775         /* bios scratch used on CIK+ */
 776         if (adev->asic_type >= CHIP_BONAIRE)
 777                 return amdgpu_atombios_scratch_need_asic_init(adev);
 778
 779         /* check MEM_SIZE for older asics */
 780         reg = amdgpu_asic_get_config_memsize(adev);
 781
 782         if ((reg != 0) && (reg != 0xffffffff))
 783                 return false;
 784
 785         return true;
 786 }
 787
 788 /**
 789  * amdgpu_dummy_page_init - init dummy page used by the driver
 790  *
 791  * @adev: amdgpu_device pointer
 792  *
 793  * Allocate the dummy page used by the driver (all asics).
 794  * This dummy page is used by the driver as a filler for gart entries
 795  * when pages are taken out of the GART
 796  * Returns 0 on sucess, -ENOMEM on failure.
 797  */
 798 int amdgpu_dummy_page_init(struct amdgpu_device *adev)
 799 {
 800         if (adev->dummy_page.page)
 801                 return 0;
 802         adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
 803         if (adev->dummy_page.page == NULL)
 804                 return -ENOMEM;
 805         adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
 806                                         0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 807         if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
 808                 dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
 809                 __free_page(adev->dummy_page.page);
 810                 adev->dummy_page.page = NULL;
 811                 return -ENOMEM;
 812         }
 813         return 0;
 814 }
 815
 816 /**
 817  * amdgpu_dummy_page_fini - free dummy page used by the driver
 818  *
 819  * @adev: amdgpu_device pointer
 820  *
 821  * Frees the dummy page used by the driver (all asics).
 822  */
 823 void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
 824 {
 825         if (adev->dummy_page.page == NULL)
 826                 return;
 827         pci_unmap_page(adev->pdev, adev->dummy_page.addr,
 828                         PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 829         __free_page(adev->dummy_page.page);
 830         adev->dummy_page.page = NULL;
 831 }
 832
 833
 834 /* ATOM accessor methods */
 835 /*
 836  * ATOM is an interpreted byte code stored in tables in the vbios.  The
 837  * driver registers callbacks to access registers and the interpreter
 838  * in the driver parses the tables and executes then to program specific
 839  * actions (set display modes, asic init, etc.).  See amdgpu_atombios.c,
 840  * atombios.h, and atom.c
 841  */
 842
 843 /**
 844  * cail_pll_read - read PLL register
 845  *
 846  * @info: atom card_info pointer
 847  * @reg: PLL register offset
 848  *
 849  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 850  * Returns the value of the PLL register.
 851  */
 852 static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
 853 {
 854         return 0;
 855 }
 856
 857 /**
 858  * cail_pll_write - write PLL register
 859  *
 860  * @info: atom card_info pointer
 861  * @reg: PLL register offset
 862  * @val: value to write to the pll register
 863  *
 864  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 865  */
 866 static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
 867 {
 868
 869 }
 870
 871 /**
 872  * cail_mc_read - read MC (Memory Controller) register
 873  *
 874  * @info: atom card_info pointer
 875  * @reg: MC register offset
 876  *
 877  * Provides an MC register accessor for the atom interpreter (r4xx+).
 878  * Returns the value of the MC register.
 879  */
 880 static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
 881 {
 882         return 0;
 883 }
 884
 885 /**
 886  * cail_mc_write - write MC (Memory Controller) register
 887  *
 888  * @info: atom card_info pointer
 889  * @reg: MC register offset
 890  * @val: value to write to the pll register
 891  *
 892  * Provides a MC register accessor for the atom interpreter (r4xx+).
 893  */
 894 static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
 895 {
 896
 897 }
 898
 899 /**
 900  * cail_reg_write - write MMIO register
 901  *
 902  * @info: atom card_info pointer
 903  * @reg: MMIO register offset
 904  * @val: value to write to the pll register
 905  *
 906  * Provides a MMIO register accessor for the atom interpreter (r4xx+).
 907  */
 908 static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
 909 {
 910         struct amdgpu_device *adev = info->dev->dev_private;
 911
 912         WREG32(reg, val);
 913 }
 914
 915 /**
 916  * cail_reg_read - read MMIO register
 917  *
 918  * @info: atom card_info pointer
 919  * @reg: MMIO register offset
 920  *
 921  * Provides an MMIO register accessor for the atom interpreter (r4xx+).
 922  * Returns the value of the MMIO register.
 923  */
 924 static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
 925 {
 926         struct amdgpu_device *adev = info->dev->dev_private;
 927         uint32_t r;
 928
 929         r = RREG32(reg);
 930         return r;
 931 }
 932
 933 /**
 934  * cail_ioreg_write - write IO register
 935  *
 936  * @info: atom card_info pointer
 937  * @reg: IO register offset
 938  * @val: value to write to the pll register
 939  *
 940  * Provides a IO register accessor for the atom interpreter (r4xx+).
 941  */
 942 static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
 943 {
 944         struct amdgpu_device *adev = info->dev->dev_private;
 945
 946         WREG32_IO(reg, val);
 947 }
 948
 949 /**
 950  * cail_ioreg_read - read IO register
 951  *
 952  * @info: atom card_info pointer
 953  * @reg: IO register offset
 954  *
 955  * Provides an IO register accessor for the atom interpreter (r4xx+).
 956  * Returns the value of the IO register.
 957  */
 958 static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
 959 {
 960         struct amdgpu_device *adev = info->dev->dev_private;
 961         uint32_t r;
 962
 963         r = RREG32_IO(reg);
 964         return r;
 965 }
 966
 967 static ssize_t amdgpu_atombios_get_vbios_version(struct device *dev,
 968                                                  struct device_attribute *attr,
 969                                                  char *buf)
 970 {
 971         struct drm_device *ddev = dev_get_drvdata(dev);
 972         struct amdgpu_device *adev = ddev->dev_private;
 973         struct atom_context *ctx = adev->mode_info.atom_context;
 974
 975         return snprintf(buf, PAGE_SIZE, "%s\n", ctx->vbios_version);
 976 }
 977
 978 static DEVICE_ATTR(vbios_version, 0444, amdgpu_atombios_get_vbios_version,
 979                    NULL);
 980
 981 /**
 982  * amdgpu_atombios_fini - free the driver info and callbacks for atombios
 983  *
 984  * @adev: amdgpu_device pointer
 985  *
 986  * Frees the driver info and register access callbacks for the ATOM
 987  * interpreter (r4xx+).
 988  * Called at driver shutdown.
 989  */
 990 static void amdgpu_atombios_fini(struct amdgpu_device *adev)
 991 {
 992         if (adev->mode_info.atom_context) {
 993                 kfree(adev->mode_info.atom_context->scratch);
 994                 kfree(adev->mode_info.atom_context->iio);
 995         }
 996         kfree(adev->mode_info.atom_context);
 997         adev->mode_info.atom_context = NULL;
 998         kfree(adev->mode_info.atom_card_info);
 999         adev->mode_info.atom_card_info = NULL;
1000         device_remove_file(adev->dev, &dev_attr_vbios_version);
1001 }
1002
1003 /**
1004  * amdgpu_atombios_init - init the driver info and callbacks for atombios
1005  *
1006  * @adev: amdgpu_device pointer
1007  *
1008  * Initializes the driver info and register access callbacks for the
1009  * ATOM interpreter (r4xx+).
1010  * Returns 0 on sucess, -ENOMEM on failure.
1011  * Called at driver startup.
1012  */
1013 static int amdgpu_atombios_init(struct amdgpu_device *adev)
1014 {
1015         struct card_info *atom_card_info =
1016             kzalloc(sizeof(struct card_info), GFP_KERNEL);
1017         int ret;
1018
1019         if (!atom_card_info)
1020                 return -ENOMEM;
1021
1022         adev->mode_info.atom_card_info = atom_card_info;
1023         atom_card_info->dev = adev->ddev;
1024         atom_card_info->reg_read = cail_reg_read;
1025         atom_card_info->reg_write = cail_reg_write;
1026         /* needed for iio ops */
1027         if (adev->rio_mem) {
1028                 atom_card_info->ioreg_read = cail_ioreg_read;
1029                 atom_card_info->ioreg_write = cail_ioreg_write;
1030         } else {
1031                 DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
1032                 atom_card_info->ioreg_read = cail_reg_read;
1033                 atom_card_info->ioreg_write = cail_reg_write;
1034         }
1035         atom_card_info->mc_read = cail_mc_read;
1036         atom_card_info->mc_write = cail_mc_write;
1037         atom_card_info->pll_read = cail_pll_read;
1038         atom_card_info->pll_write = cail_pll_write;
1039
1040         adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
1041         if (!adev->mode_info.atom_context) {
1042                 amdgpu_atombios_fini(adev);
1043                 return -ENOMEM;
1044         }
1045
1046         mutex_init(&adev->mode_info.atom_context->mutex);
1047         if (adev->is_atom_fw) {
1048                 amdgpu_atomfirmware_scratch_regs_init(adev);
1049                 amdgpu_atomfirmware_allocate_fb_scratch(adev);
1050         } else {
1051                 amdgpu_atombios_scratch_regs_init(adev);
1052                 amdgpu_atombios_allocate_fb_scratch(adev);
1053         }
1054
1055         ret = device_create_file(adev->dev, &dev_attr_vbios_version);
1056         if (ret) {
1057                 DRM_ERROR("Failed to create device file for VBIOS version\n");
1058                 return ret;
1059         }
1060
1061         return 0;
1062 }
1063
1064 /* if we get transitioned to only one device, take VGA back */
1065 /**
1066  * amdgpu_vga_set_decode - enable/disable vga decode
1067  *
1068  * @cookie: amdgpu_device pointer
1069  * @state: enable/disable vga decode
1070  *
1071  * Enable/disable vga decode (all asics).
1072  * Returns VGA resource flags.
1073  */
1074 static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
1075 {
1076         struct amdgpu_device *adev = cookie;
1077         amdgpu_asic_set_vga_state(adev, state);
1078         if (state)
1079                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1080                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1081         else
1082                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1083 }
1084
1085 static void amdgpu_check_block_size(struct amdgpu_device *adev)
1086 {
1087         /* defines number of bits in page table versus page directory,
1088          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1089          * page table and the remaining bits are in the page directory */
1090         if (amdgpu_vm_block_size == -1)
1091                 return;
1092
1093         if (amdgpu_vm_block_size < 9) {
1094                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1095                          amdgpu_vm_block_size);
1096                 goto def_value;
1097         }
1098
1099         if (amdgpu_vm_block_size > 24 ||
1100             (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
1101                 dev_warn(adev->dev, "VM page table size (%d) too large\n",
1102                          amdgpu_vm_block_size);
1103                 goto def_value;
1104         }
1105
1106         return;
1107
1108 def_value:
1109         amdgpu_vm_block_size = -1;
1110 }
1111
1112 static void amdgpu_check_vm_size(struct amdgpu_device *adev)
1113 {
1114         /* no need to check the default value */
1115         if (amdgpu_vm_size == -1)
1116                 return;
1117
1118         if (!is_power_of_2(amdgpu_vm_size)) {
1119                 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
1120                          amdgpu_vm_size);
1121                 goto def_value;
1122         }
1123
1124         if (amdgpu_vm_size < 1) {
1125                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1126                          amdgpu_vm_size);
1127                 goto def_value;
1128         }
1129
1130         /*
1131          * Max GPUVM size for Cayman, SI, CI VI are 40 bits.
1132          */
1133         if (amdgpu_vm_size > 1024) {
1134                 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
1135                          amdgpu_vm_size);
1136                 goto def_value;
1137         }
1138
1139         return;
1140
1141 def_value:
1142         amdgpu_vm_size = -1;
1143 }
1144
1145 /**
1146  * amdgpu_check_arguments - validate module params
1147  *
1148  * @adev: amdgpu_device pointer
1149  *
1150  * Validates certain module parameters and updates
1151  * the associated values used by the driver (all asics).
1152  */
1153 static void amdgpu_check_arguments(struct amdgpu_device *adev)
1154 {
1155         if (amdgpu_sched_jobs < 4) {
1156                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1157                          amdgpu_sched_jobs);
1158                 amdgpu_sched_jobs = 4;
1159         } else if (!is_power_of_2(amdgpu_sched_jobs)){
1160                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1161                          amdgpu_sched_jobs);
1162                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1163         }
1164
1165         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1166                 /* gart size must be greater or equal to 32M */
1167                 dev_warn(adev->dev, "gart size (%d) too small\n",
1168                          amdgpu_gart_size);
1169                 amdgpu_gart_size = -1;
1170         }
1171
1172         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1173                 /* gtt size must be greater or equal to 32M */
1174                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1175                                  amdgpu_gtt_size);
1176                 amdgpu_gtt_size = -1;
1177         }
1178
1179         /* valid range is between 4 and 9 inclusive */
1180         if (amdgpu_vm_fragment_size != -1 &&
1181             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1182                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1183                 amdgpu_vm_fragment_size = -1;
1184         }
1185
1186         amdgpu_check_vm_size(adev);
1187
1188         amdgpu_check_block_size(adev);
1189
1190         if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
1191             !is_power_of_2(amdgpu_vram_page_split))) {
1192                 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
1193                          amdgpu_vram_page_split);
1194                 amdgpu_vram_page_split = 1024;
1195         }
1196 }
1197
1198 /**
1199  * amdgpu_switcheroo_set_state - set switcheroo state
1200  *
1201  * @pdev: pci dev pointer
1202  * @state: vga_switcheroo state
1203  *
1204  * Callback for the switcheroo driver.  Suspends or resumes the
1205  * the asics before or after it is powered up using ACPI methods.
1206  */
1207 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1208 {
1209         struct drm_device *dev = pci_get_drvdata(pdev);
1210
1211         if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1212                 return;
1213
1214         if (state == VGA_SWITCHEROO_ON) {
1215                 pr_info("amdgpu: switched on\n");
1216                 /* don't suspend or resume card normally */
1217                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1218
1219                 amdgpu_device_resume(dev, true, true);
1220
1221                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1222                 drm_kms_helper_poll_enable(dev);
1223         } else {
1224                 pr_info("amdgpu: switched off\n");
1225                 drm_kms_helper_poll_disable(dev);
1226                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1227                 amdgpu_device_suspend(dev, true, true);
1228                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1229         }
1230 }
1231
1232 /**
1233  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1234  *
1235  * @pdev: pci dev pointer
1236  *
1237  * Callback for the switcheroo driver.  Check of the switcheroo
1238  * state can be changed.
1239  * Returns true if the state can be changed, false if not.
1240  */
1241 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1242 {
1243         struct drm_device *dev = pci_get_drvdata(pdev);
1244
1245         /*
1246         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1247         * locking inversion with the driver load path. And the access here is
1248         * completely racy anyway. So don't bother with locking for now.
1249         */
1250         return dev->open_count == 0;
1251 }
1252
1253 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1254         .set_gpu_state = amdgpu_switcheroo_set_state,
1255         .reprobe = NULL,
1256         .can_switch = amdgpu_switcheroo_can_switch,
1257 };
1258
1259 int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
1260                                   enum amd_ip_block_type block_type,
1261                                   enum amd_clockgating_state state)
1262 {
1263         int i, r = 0;
1264
1265         for (i = 0; i < adev->num_ip_blocks; i++) {
1266                 if (!adev->ip_blocks[i].status.valid)
1267                         continue;
1268                 if (adev->ip_blocks[i].version->type != block_type)
1269                         continue;
1270                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1271                         continue;
1272                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1273                         (void *)adev, state);
1274                 if (r)
1275                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1276                                   adev->ip_blocks[i].version->funcs->name, r);
1277         }
1278         return r;
1279 }
1280
1281 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
1282                                   enum amd_ip_block_type block_type,
1283                                   enum amd_powergating_state state)
1284 {
1285         int i, r = 0;
1286
1287         for (i = 0; i < adev->num_ip_blocks; i++) {
1288                 if (!adev->ip_blocks[i].status.valid)
1289                         continue;
1290                 if (adev->ip_blocks[i].version->type != block_type)
1291                         continue;
1292                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1293                         continue;
1294                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1295                         (void *)adev, state);
1296                 if (r)
1297                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1298                                   adev->ip_blocks[i].version->funcs->name, r);
1299         }
1300         return r;
1301 }
1302
1303 void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
1304 {
1305         int i;
1306
1307         for (i = 0; i < adev->num_ip_blocks; i++) {
1308                 if (!adev->ip_blocks[i].status.valid)
1309                         continue;
1310                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1311                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1312         }
1313 }
1314
1315 int amdgpu_wait_for_idle(struct amdgpu_device *adev,
1316                          enum amd_ip_block_type block_type)
1317 {
1318         int i, r;
1319
1320         for (i = 0; i < adev->num_ip_blocks; i++) {
1321                 if (!adev->ip_blocks[i].status.valid)
1322                         continue;
1323                 if (adev->ip_blocks[i].version->type == block_type) {
1324                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1325                         if (r)
1326                                 return r;
1327                         break;
1328                 }
1329         }
1330         return 0;
1331
1332 }
1333
1334 bool amdgpu_is_idle(struct amdgpu_device *adev,
1335                     enum amd_ip_block_type block_type)
1336 {
1337         int i;
1338
1339         for (i = 0; i < adev->num_ip_blocks; i++) {
1340                 if (!adev->ip_blocks[i].status.valid)
1341                         continue;
1342                 if (adev->ip_blocks[i].version->type == block_type)
1343                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1344         }
1345         return true;
1346
1347 }
1348
1349 struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
1350                                              enum amd_ip_block_type type)
1351 {
1352         int i;
1353
1354         for (i = 0; i < adev->num_ip_blocks; i++)
1355                 if (adev->ip_blocks[i].version->type == type)
1356                         return &adev->ip_blocks[i];
1357
1358         return NULL;
1359 }
1360
1361 /**
1362  * amdgpu_ip_block_version_cmp
1363  *
1364  * @adev: amdgpu_device pointer
1365  * @type: enum amd_ip_block_type
1366  * @major: major version
1367  * @minor: minor version
1368  *
1369  * return 0 if equal or greater
1370  * return 1 if smaller or the ip_block doesn't exist
1371  */
1372 int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
1373                                 enum amd_ip_block_type type,
1374                                 u32 major, u32 minor)
1375 {
1376         struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
1377
1378         if (ip_block && ((ip_block->version->major > major) ||
1379                         ((ip_block->version->major == major) &&
1380                         (ip_block->version->minor >= minor))))
1381                 return 0;
1382
1383         return 1;
1384 }
1385
1386 /**
1387  * amdgpu_ip_block_add
1388  *
1389  * @adev: amdgpu_device pointer
1390  * @ip_block_version: pointer to the IP to add
1391  *
1392  * Adds the IP block driver information to the collection of IPs
1393  * on the asic.
1394  */
1395 int amdgpu_ip_block_add(struct amdgpu_device *adev,
1396                         const struct amdgpu_ip_block_version *ip_block_version)
1397 {
1398         if (!ip_block_version)
1399                 return -EINVAL;
1400
1401         DRM_DEBUG("add ip block number %d <%s>\n", adev->num_ip_blocks,
1402                   ip_block_version->funcs->name);
1403
1404         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1405
1406         return 0;
1407 }
1408
1409 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1410 {
1411         adev->enable_virtual_display = false;
1412
1413         if (amdgpu_virtual_display) {
1414                 struct drm_device *ddev = adev->ddev;
1415                 const char *pci_address_name = pci_name(ddev->pdev);
1416                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1417
1418                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1419                 pciaddstr_tmp = pciaddstr;
1420                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1421                         pciaddname = strsep(&pciaddname_tmp, ",");
1422                         if (!strcmp("all", pciaddname)
1423                             || !strcmp(pci_address_name, pciaddname)) {
1424                                 long num_crtc;
1425                                 int res = -1;
1426
1427                                 adev->enable_virtual_display = true;
1428
1429                                 if (pciaddname_tmp)
1430                                         res = kstrtol(pciaddname_tmp, 10,
1431                                                       &num_crtc);
1432
1433                                 if (!res) {
1434                                         if (num_crtc < 1)
1435                                                 num_crtc = 1;
1436                                         if (num_crtc > 6)
1437                                                 num_crtc = 6;
1438                                         adev->mode_info.num_crtc = num_crtc;
1439                                 } else {
1440                                         adev->mode_info.num_crtc = 1;
1441                                 }
1442                                 break;
1443                         }
1444                 }
1445
1446                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1447                          amdgpu_virtual_display, pci_address_name,
1448                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1449
1450                 kfree(pciaddstr);
1451         }
1452 }
1453
1454 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1455 {
1456         const char *chip_name;
1457         char fw_name[30];
1458         int err;
1459         const struct gpu_info_firmware_header_v1_0 *hdr;
1460
1461         adev->firmware.gpu_info_fw = NULL;
1462
1463         switch (adev->asic_type) {
1464         case CHIP_TOPAZ:
1465         case CHIP_TONGA:
1466         case CHIP_FIJI:
1467         case CHIP_POLARIS11:
1468         case CHIP_POLARIS10:
1469         case CHIP_POLARIS12:
1470         case CHIP_CARRIZO:
1471         case CHIP_STONEY:
1472 #ifdef CONFIG_DRM_AMDGPU_SI
1473         case CHIP_VERDE:
1474         case CHIP_TAHITI:
1475         case CHIP_PITCAIRN:
1476         case CHIP_OLAND:
1477         case CHIP_HAINAN:
1478 #endif
1479 #ifdef CONFIG_DRM_AMDGPU_CIK
1480         case CHIP_BONAIRE:
1481         case CHIP_HAWAII:
1482         case CHIP_KAVERI:
1483         case CHIP_KABINI:
1484         case CHIP_MULLINS:
1485 #endif
1486         default:
1487                 return 0;
1488         case CHIP_VEGA10:
1489                 chip_name = "vega10";
1490                 break;
1491         case CHIP_RAVEN:
1492                 chip_name = "raven";
1493                 break;
1494         }
1495
1496         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1497         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1498         if (err) {
1499                 dev_err(adev->dev,
1500                         "Failed to load gpu_info firmware \"%s\"\n",
1501                         fw_name);
1502                 goto out;
1503         }
1504         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1505         if (err) {
1506                 dev_err(adev->dev,
1507                         "Failed to validate gpu_info firmware \"%s\"\n",
1508                         fw_name);
1509                 goto out;
1510         }
1511
1512         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1513         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1514
1515         switch (hdr->version_major) {
1516         case 1:
1517         {
1518                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1519                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1520                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1521
1522                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1523                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1524                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1525                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1526                 adev->gfx.config.max_texture_channel_caches =
1527                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
1528                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1529                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1530                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1531                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1532                 adev->gfx.config.double_offchip_lds_buf =
1533                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1534                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1535                 adev->gfx.cu_info.max_waves_per_simd =
1536                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1537                 adev->gfx.cu_info.max_scratch_slots_per_cu =
1538                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1539                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1540                 break;
1541         }
1542         default:
1543                 dev_err(adev->dev,
1544                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1545                 err = -EINVAL;
1546                 goto out;
1547         }
1548 out:
1549         return err;
1550 }
1551
1552 static int amdgpu_early_init(struct amdgpu_device *adev)
1553 {
1554         int i, r;
1555
1556         amdgpu_device_enable_virtual_display(adev);
1557
1558         switch (adev->asic_type) {
1559         case CHIP_TOPAZ:
1560         case CHIP_TONGA:
1561         case CHIP_FIJI:
1562         case CHIP_POLARIS11:
1563         case CHIP_POLARIS10:
1564         case CHIP_POLARIS12:
1565         case CHIP_CARRIZO:
1566         case CHIP_STONEY:
1567                 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1568                         adev->family = AMDGPU_FAMILY_CZ;
1569                 else
1570                         adev->family = AMDGPU_FAMILY_VI;
1571
1572                 r = vi_set_ip_blocks(adev);
1573                 if (r)
1574                         return r;
1575                 break;
1576 #ifdef CONFIG_DRM_AMDGPU_SI
1577         case CHIP_VERDE:
1578         case CHIP_TAHITI:
1579         case CHIP_PITCAIRN:
1580         case CHIP_OLAND:
1581         case CHIP_HAINAN:
1582                 adev->family = AMDGPU_FAMILY_SI;
1583                 r = si_set_ip_blocks(adev);
1584                 if (r)
1585                         return r;
1586                 break;
1587 #endif
1588 #ifdef CONFIG_DRM_AMDGPU_CIK
1589         case CHIP_BONAIRE:
1590         case CHIP_HAWAII:
1591         case CHIP_KAVERI:
1592         case CHIP_KABINI:
1593         case CHIP_MULLINS:
1594                 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1595                         adev->family = AMDGPU_FAMILY_CI;
1596                 else
1597                         adev->family = AMDGPU_FAMILY_KV;
1598
1599                 r = cik_set_ip_blocks(adev);
1600                 if (r)
1601                         return r;
1602                 break;
1603 #endif
1604         case  CHIP_VEGA10:
1605         case  CHIP_RAVEN:
1606                 if (adev->asic_type == CHIP_RAVEN)
1607                         adev->family = AMDGPU_FAMILY_RV;
1608                 else
1609                         adev->family = AMDGPU_FAMILY_AI;
1610
1611                 r = soc15_set_ip_blocks(adev);
1612                 if (r)
1613                         return r;
1614                 break;
1615         default:
1616                 /* FIXME: not supported yet */
1617                 return -EINVAL;
1618         }
1619
1620         r = amdgpu_device_parse_gpu_info_fw(adev);
1621         if (r)
1622                 return r;
1623
1624         if (amdgpu_sriov_vf(adev)) {
1625                 r = amdgpu_virt_request_full_gpu(adev, true);
1626                 if (r)
1627                         return r;
1628         }
1629
1630         for (i = 0; i < adev->num_ip_blocks; i++) {
1631                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1632                         DRM_ERROR("disabled ip block: %d <%s>\n",
1633                                   i, adev->ip_blocks[i].version->funcs->name);
1634                         adev->ip_blocks[i].status.valid = false;
1635                 } else {
1636                         if (adev->ip_blocks[i].version->funcs->early_init) {
1637                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1638                                 if (r == -ENOENT) {
1639                                         adev->ip_blocks[i].status.valid = false;
1640                                 } else if (r) {
1641                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
1642                                                   adev->ip_blocks[i].version->funcs->name, r);
1643                                         return r;
1644                                 } else {
1645                                         adev->ip_blocks[i].status.valid = true;
1646                                 }
1647                         } else {
1648                                 adev->ip_blocks[i].status.valid = true;
1649                         }
1650                 }
1651         }
1652
1653         adev->cg_flags &= amdgpu_cg_mask;
1654         adev->pg_flags &= amdgpu_pg_mask;
1655
1656         return 0;
1657 }
1658
1659 static int amdgpu_init(struct amdgpu_device *adev)
1660 {
1661         int i, r;
1662
1663         for (i = 0; i < adev->num_ip_blocks; i++) {
1664                 if (!adev->ip_blocks[i].status.valid)
1665                         continue;
1666                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1667                 if (r) {
1668                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1669                                   adev->ip_blocks[i].version->funcs->name, r);
1670                         return r;
1671                 }
1672                 adev->ip_blocks[i].status.sw = true;
1673                 /* need to do gmc hw init early so we can allocate gpu mem */
1674                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1675                         r = amdgpu_vram_scratch_init(adev);
1676                         if (r) {
1677                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1678                                 return r;
1679                         }
1680                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1681                         if (r) {
1682                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
1683                                 return r;
1684                         }
1685                         r = amdgpu_wb_init(adev);
1686                         if (r) {
1687                                 DRM_ERROR("amdgpu_wb_init failed %d\n", r);
1688                                 return r;
1689                         }
1690                         adev->ip_blocks[i].status.hw = true;
1691
1692                         /* right after GMC hw init, we create CSA */
1693                         if (amdgpu_sriov_vf(adev)) {
1694                                 r = amdgpu_allocate_static_csa(adev);
1695                                 if (r) {
1696                                         DRM_ERROR("allocate CSA failed %d\n", r);
1697                                         return r;
1698                                 }
1699                         }
1700                 }
1701         }
1702
1703         for (i = 0; i < adev->num_ip_blocks; i++) {
1704                 if (!adev->ip_blocks[i].status.sw)
1705                         continue;
1706                 /* gmc hw init is done early */
1707                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
1708                         continue;
1709                 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1710                 if (r) {
1711                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1712                                   adev->ip_blocks[i].version->funcs->name, r);
1713                         return r;
1714                 }
1715                 adev->ip_blocks[i].status.hw = true;
1716         }
1717
1718         return 0;
1719 }
1720
1721 static void amdgpu_fill_reset_magic(struct amdgpu_device *adev)
1722 {
1723         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1724 }
1725
1726 static bool amdgpu_check_vram_lost(struct amdgpu_device *adev)
1727 {
1728         return !!memcmp(adev->gart.ptr, adev->reset_magic,
1729                         AMDGPU_RESET_MAGIC_NUM);
1730 }
1731
1732 static int amdgpu_late_set_cg_state(struct amdgpu_device *adev)
1733 {
1734         int i = 0, r;
1735
1736         for (i = 0; i < adev->num_ip_blocks; i++) {
1737                 if (!adev->ip_blocks[i].status.valid)
1738                         continue;
1739                 /* skip CG for VCE/UVD, it's handled specially */
1740                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1741                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1742                         /* enable clockgating to save power */
1743                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1744                                                                                      AMD_CG_STATE_GATE);
1745                         if (r) {
1746                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1747                                           adev->ip_blocks[i].version->funcs->name, r);
1748                                 return r;
1749                         }
1750                 }
1751         }
1752         return 0;
1753 }
1754
1755 static int amdgpu_late_init(struct amdgpu_device *adev)
1756 {
1757         int i = 0, r;
1758
1759         for (i = 0; i < adev->num_ip_blocks; i++) {
1760                 if (!adev->ip_blocks[i].status.valid)
1761                         continue;
1762                 if (adev->ip_blocks[i].version->funcs->late_init) {
1763                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1764                         if (r) {
1765                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1766                                           adev->ip_blocks[i].version->funcs->name, r);
1767                                 return r;
1768                         }
1769                         adev->ip_blocks[i].status.late_initialized = true;
1770                 }
1771         }
1772
1773         mod_delayed_work(system_wq, &adev->late_init_work,
1774                         msecs_to_jiffies(AMDGPU_RESUME_MS));
1775
1776         amdgpu_fill_reset_magic(adev);
1777
1778         return 0;
1779 }
1780
1781 static int amdgpu_fini(struct amdgpu_device *adev)
1782 {
1783         int i, r;
1784
1785         /* need to disable SMC first */
1786         for (i = 0; i < adev->num_ip_blocks; i++) {
1787                 if (!adev->ip_blocks[i].status.hw)
1788                         continue;
1789                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
1790                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1791                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1792                                                                                      AMD_CG_STATE_UNGATE);
1793                         if (r) {
1794                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1795                                           adev->ip_blocks[i].version->funcs->name, r);
1796                                 return r;
1797                         }
1798                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1799                         /* XXX handle errors */
1800                         if (r) {
1801                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1802                                           adev->ip_blocks[i].version->funcs->name, r);
1803                         }
1804                         adev->ip_blocks[i].status.hw = false;
1805                         break;
1806                 }
1807         }
1808
1809         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1810                 if (!adev->ip_blocks[i].status.hw)
1811                         continue;
1812                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1813                         amdgpu_wb_fini(adev);
1814                         amdgpu_vram_scratch_fini(adev);
1815                 }
1816
1817                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1818                         adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1819                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1820                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1821                                                                                      AMD_CG_STATE_UNGATE);
1822                         if (r) {
1823                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1824                                           adev->ip_blocks[i].version->funcs->name, r);
1825                                 return r;
1826                         }
1827                 }
1828
1829                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1830                 /* XXX handle errors */
1831                 if (r) {
1832                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1833                                   adev->ip_blocks[i].version->funcs->name, r);
1834                 }
1835
1836                 adev->ip_blocks[i].status.hw = false;
1837         }
1838
1839         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1840                 if (!adev->ip_blocks[i].status.sw)
1841                         continue;
1842                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1843                 /* XXX handle errors */
1844                 if (r) {
1845                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1846                                   adev->ip_blocks[i].version->funcs->name, r);
1847                 }
1848                 adev->ip_blocks[i].status.sw = false;
1849                 adev->ip_blocks[i].status.valid = false;
1850         }
1851
1852         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1853                 if (!adev->ip_blocks[i].status.late_initialized)
1854                         continue;
1855                 if (adev->ip_blocks[i].version->funcs->late_fini)
1856                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1857                 adev->ip_blocks[i].status.late_initialized = false;
1858         }
1859
1860         if (amdgpu_sriov_vf(adev))
1861                 amdgpu_virt_release_full_gpu(adev, false);
1862
1863         return 0;
1864 }
1865
1866 static void amdgpu_late_init_func_handler(struct work_struct *work)
1867 {
1868         struct amdgpu_device *adev =
1869                 container_of(work, struct amdgpu_device, late_init_work.work);
1870         amdgpu_late_set_cg_state(adev);
1871 }
1872
1873 int amdgpu_suspend(struct amdgpu_device *adev)
1874 {
1875         int i, r;
1876
1877         if (amdgpu_sriov_vf(adev))
1878                 amdgpu_virt_request_full_gpu(adev, false);
1879
1880         /* ungate SMC block first */
1881         r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1882                                          AMD_CG_STATE_UNGATE);
1883         if (r) {
1884                 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
1885         }
1886
1887         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1888                 if (!adev->ip_blocks[i].status.valid)
1889                         continue;
1890                 /* ungate blocks so that suspend can properly shut them down */
1891                 if (i != AMD_IP_BLOCK_TYPE_SMC) {
1892                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1893                                                                                      AMD_CG_STATE_UNGATE);
1894                         if (r) {
1895                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1896                                           adev->ip_blocks[i].version->funcs->name, r);
1897                         }
1898                 }
1899                 /* XXX handle errors */
1900                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1901                 /* XXX handle errors */
1902                 if (r) {
1903                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
1904                                   adev->ip_blocks[i].version->funcs->name, r);
1905                 }
1906         }
1907
1908         if (amdgpu_sriov_vf(adev))
1909                 amdgpu_virt_release_full_gpu(adev, false);
1910
1911         return 0;
1912 }
1913
1914 static int amdgpu_sriov_reinit_early(struct amdgpu_device *adev)
1915 {
1916         int i, r;
1917
1918         static enum amd_ip_block_type ip_order[] = {
1919                 AMD_IP_BLOCK_TYPE_GMC,
1920                 AMD_IP_BLOCK_TYPE_COMMON,
1921                 AMD_IP_BLOCK_TYPE_IH,
1922         };
1923
1924         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1925                 int j;
1926                 struct amdgpu_ip_block *block;
1927
1928                 for (j = 0; j < adev->num_ip_blocks; j++) {
1929                         block = &adev->ip_blocks[j];
1930
1931                         if (block->version->type != ip_order[i] ||
1932                                 !block->status.valid)
1933                                 continue;
1934
1935                         r = block->version->funcs->hw_init(adev);
1936                         DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1937                 }
1938         }
1939
1940         return 0;
1941 }
1942
1943 static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
1944 {
1945         int i, r;
1946
1947         static enum amd_ip_block_type ip_order[] = {
1948                 AMD_IP_BLOCK_TYPE_SMC,
1949                 AMD_IP_BLOCK_TYPE_PSP,
1950                 AMD_IP_BLOCK_TYPE_DCE,
1951                 AMD_IP_BLOCK_TYPE_GFX,
1952                 AMD_IP_BLOCK_TYPE_SDMA,
1953                 AMD_IP_BLOCK_TYPE_UVD,
1954                 AMD_IP_BLOCK_TYPE_VCE
1955         };
1956
1957         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1958                 int j;
1959                 struct amdgpu_ip_block *block;
1960
1961                 for (j = 0; j < adev->num_ip_blocks; j++) {
1962                         block = &adev->ip_blocks[j];
1963
1964                         if (block->version->type != ip_order[i] ||
1965                                 !block->status.valid)
1966                                 continue;
1967
1968                         r = block->version->funcs->hw_init(adev);
1969                         DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1970                 }
1971         }
1972
1973         return 0;
1974 }
1975
1976 static int amdgpu_resume_phase1(struct amdgpu_device *adev)
1977 {
1978         int i, r;
1979
1980         for (i = 0; i < adev->num_ip_blocks; i++) {
1981                 if (!adev->ip_blocks[i].status.valid)
1982                         continue;
1983                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1984                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1985                                 adev->ip_blocks[i].version->type ==
1986                                 AMD_IP_BLOCK_TYPE_IH) {
1987                         r = adev->ip_blocks[i].version->funcs->resume(adev);
1988                         if (r) {
1989                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
1990                                           adev->ip_blocks[i].version->funcs->name, r);
1991                                 return r;
1992                         }
1993                 }
1994         }
1995
1996         return 0;
1997 }
1998
1999 static int amdgpu_resume_phase2(struct amdgpu_device *adev)
2000 {
2001         int i, r;
2002
2003         for (i = 0; i < adev->num_ip_blocks; i++) {
2004                 if (!adev->ip_blocks[i].status.valid)
2005                         continue;
2006                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2007                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2008                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH )
2009                         continue;
2010                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2011                 if (r) {
2012                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2013                                   adev->ip_blocks[i].version->funcs->name, r);
2014                         return r;
2015                 }
2016         }
2017
2018         return 0;
2019 }
2020
2021 static int amdgpu_resume(struct amdgpu_device *adev)
2022 {
2023         int r;
2024
2025         r = amdgpu_resume_phase1(adev);
2026         if (r)
2027                 return r;
2028         r = amdgpu_resume_phase2(adev);
2029
2030         return r;
2031 }
2032
2033 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2034 {
2035         if (amdgpu_sriov_vf(adev)) {
2036                 if (adev->is_atom_fw) {
2037                         if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2038                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2039                 } else {
2040                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2041                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2042                 }
2043
2044                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
2045                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2046         }
2047 }
2048
2049 /**
2050  * amdgpu_device_init - initialize the driver
2051  *
2052  * @adev: amdgpu_device pointer
2053  * @pdev: drm dev pointer
2054  * @pdev: pci dev pointer
2055  * @flags: driver flags
2056  *
2057  * Initializes the driver info and hw (all asics).
2058  * Returns 0 for success or an error on failure.
2059  * Called at driver startup.
2060  */
2061 int amdgpu_device_init(struct amdgpu_device *adev,
2062                        struct drm_device *ddev,
2063                        struct pci_dev *pdev,
2064                        uint32_t flags)
2065 {
2066         int r, i;
2067         bool runtime = false;
2068         u32 max_MBps;
2069
2070         adev->shutdown = false;
2071         adev->dev = &pdev->dev;
2072         adev->ddev = ddev;
2073         adev->pdev = pdev;
2074         adev->flags = flags;
2075         adev->asic_type = flags & AMD_ASIC_MASK;
2076         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2077         adev->mc.gart_size = 512 * 1024 * 1024;
2078         adev->accel_working = false;
2079         adev->num_rings = 0;
2080         adev->mman.buffer_funcs = NULL;
2081         adev->mman.buffer_funcs_ring = NULL;
2082         adev->vm_manager.vm_pte_funcs = NULL;
2083         adev->vm_manager.vm_pte_num_rings = 0;
2084         adev->gart.gart_funcs = NULL;
2085         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2086         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2087
2088         adev->smc_rreg = &amdgpu_invalid_rreg;
2089         adev->smc_wreg = &amdgpu_invalid_wreg;
2090         adev->pcie_rreg = &amdgpu_invalid_rreg;
2091         adev->pcie_wreg = &amdgpu_invalid_wreg;
2092         adev->pciep_rreg = &amdgpu_invalid_rreg;
2093         adev->pciep_wreg = &amdgpu_invalid_wreg;
2094         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2095         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2096         adev->didt_rreg = &amdgpu_invalid_rreg;
2097         adev->didt_wreg = &amdgpu_invalid_wreg;
2098         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2099         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
2100         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2101         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2102
2103
2104         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2105                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2106                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
2107
2108         /* mutex initialization are all done here so we
2109          * can recall function without having locking issues */
2110         atomic_set(&adev->irq.ih.lock, 0);
2111         mutex_init(&adev->firmware.mutex);
2112         mutex_init(&adev->pm.mutex);
2113         mutex_init(&adev->gfx.gpu_clock_mutex);
2114         mutex_init(&adev->srbm_mutex);
2115         mutex_init(&adev->gfx.pipe_reserve_mutex);
2116         mutex_init(&adev->grbm_idx_mutex);
2117         mutex_init(&adev->mn_lock);
2118         mutex_init(&adev->virt.vf_errors.lock);
2119         hash_init(adev->mn_hash);
2120
2121         amdgpu_check_arguments(adev);
2122
2123         spin_lock_init(&adev->mmio_idx_lock);
2124         spin_lock_init(&adev->smc_idx_lock);
2125         spin_lock_init(&adev->pcie_idx_lock);
2126         spin_lock_init(&adev->uvd_ctx_idx_lock);
2127         spin_lock_init(&adev->didt_idx_lock);
2128         spin_lock_init(&adev->gc_cac_idx_lock);
2129         spin_lock_init(&adev->se_cac_idx_lock);
2130         spin_lock_init(&adev->audio_endpt_idx_lock);
2131         spin_lock_init(&adev->mm_stats.lock);
2132
2133         INIT_LIST_HEAD(&adev->shadow_list);
2134         mutex_init(&adev->shadow_list_lock);
2135
2136         INIT_LIST_HEAD(&adev->gtt_list);
2137         spin_lock_init(&adev->gtt_list_lock);
2138
2139         INIT_LIST_HEAD(&adev->ring_lru_list);
2140         spin_lock_init(&adev->ring_lru_list_lock);
2141
2142         INIT_DELAYED_WORK(&adev->late_init_work, amdgpu_late_init_func_handler);
2143
2144         /* Registers mapping */
2145         /* TODO: block userspace mapping of io register */
2146         if (adev->asic_type >= CHIP_BONAIRE) {
2147                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2148                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2149         } else {
2150                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2151                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2152         }
2153
2154         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2155         if (adev->rmmio == NULL) {
2156                 return -ENOMEM;
2157         }
2158         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2159         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2160
2161         /* doorbell bar mapping */
2162         amdgpu_doorbell_init(adev);
2163
2164         /* io port mapping */
2165         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2166                 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2167                         adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2168                         adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2169                         break;
2170                 }
2171         }
2172         if (adev->rio_mem == NULL)
2173                 DRM_INFO("PCI I/O BAR is not found.\n");
2174
2175         /* early init functions */
2176         r = amdgpu_early_init(adev);
2177         if (r)
2178                 return r;
2179
2180         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2181         /* this will fail for cards that aren't VGA class devices, just
2182          * ignore it */
2183         vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
2184
2185         if (amdgpu_runtime_pm == 1)
2186                 runtime = true;
2187         if (amdgpu_device_is_px(ddev))
2188                 runtime = true;
2189         if (!pci_is_thunderbolt_attached(adev->pdev))
2190                 vga_switcheroo_register_client(adev->pdev,
2191                                                &amdgpu_switcheroo_ops, runtime);
2192         if (runtime)
2193                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2194
2195         /* Read BIOS */
2196         if (!amdgpu_get_bios(adev)) {
2197                 r = -EINVAL;
2198                 goto failed;
2199         }
2200
2201         r = amdgpu_atombios_init(adev);
2202         if (r) {
2203                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2204                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2205                 goto failed;
2206         }
2207
2208         /* detect if we are with an SRIOV vbios */
2209         amdgpu_device_detect_sriov_bios(adev);
2210
2211         /* Post card if necessary */
2212         if (amdgpu_need_post(adev)) {
2213                 if (!adev->bios) {
2214                         dev_err(adev->dev, "no vBIOS found\n");
2215                         r = -EINVAL;
2216                         goto failed;
2217                 }
2218                 DRM_INFO("GPU posting now...\n");
2219                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2220                 if (r) {
2221                         dev_err(adev->dev, "gpu post error!\n");
2222                         goto failed;
2223                 }
2224         } else {
2225                 DRM_INFO("GPU post is not needed\n");
2226         }
2227
2228         if (adev->is_atom_fw) {
2229                 /* Initialize clocks */
2230                 r = amdgpu_atomfirmware_get_clock_info(adev);
2231                 if (r) {
2232                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
2233                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2234                         goto failed;
2235                 }
2236         } else {
2237                 /* Initialize clocks */
2238                 r = amdgpu_atombios_get_clock_info(adev);
2239                 if (r) {
2240                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2241                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2242                         goto failed;
2243                 }
2244                 /* init i2c buses */
2245                 amdgpu_atombios_i2c_init(adev);
2246         }
2247
2248         /* Fence driver */
2249         r = amdgpu_fence_driver_init(adev);
2250         if (r) {
2251                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2252                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2253                 goto failed;
2254         }
2255
2256         /* init the mode config */
2257         drm_mode_config_init(adev->ddev);
2258
2259         r = amdgpu_init(adev);
2260         if (r) {
2261                 dev_err(adev->dev, "amdgpu_init failed\n");
2262                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2263                 amdgpu_fini(adev);
2264                 goto failed;
2265         }
2266
2267         adev->accel_working = true;
2268
2269         amdgpu_vm_check_compute_bug(adev);
2270
2271         /* Initialize the buffer migration limit. */
2272         if (amdgpu_moverate >= 0)
2273                 max_MBps = amdgpu_moverate;
2274         else
2275                 max_MBps = 8; /* Allow 8 MB/s. */
2276         /* Get a log2 for easy divisions. */
2277         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2278
2279         r = amdgpu_ib_pool_init(adev);
2280         if (r) {
2281                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2282                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2283                 goto failed;
2284         }
2285
2286         r = amdgpu_ib_ring_tests(adev);
2287         if (r)
2288                 DRM_ERROR("ib ring test failed (%d).\n", r);
2289
2290         if (amdgpu_sriov_vf(adev))
2291                 amdgpu_virt_init_data_exchange(adev);
2292
2293         amdgpu_fbdev_init(adev);
2294
2295         r = amdgpu_pm_sysfs_init(adev);
2296         if (r)
2297                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2298
2299         r = amdgpu_gem_debugfs_init(adev);
2300         if (r)
2301                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2302
2303         r = amdgpu_debugfs_regs_init(adev);
2304         if (r)
2305                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2306
2307         r = amdgpu_debugfs_test_ib_ring_init(adev);
2308         if (r)
2309                 DRM_ERROR("registering register test ib ring debugfs failed (%d).\n", r);
2310
2311         r = amdgpu_debugfs_firmware_init(adev);
2312         if (r)
2313                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2314
2315         r = amdgpu_debugfs_vbios_dump_init(adev);
2316         if (r)
2317                 DRM_ERROR("Creating vbios dump debugfs failed (%d).\n", r);
2318
2319         if ((amdgpu_testing & 1)) {
2320                 if (adev->accel_working)
2321                         amdgpu_test_moves(adev);
2322                 else
2323                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2324         }
2325         if (amdgpu_benchmarking) {
2326                 if (adev->accel_working)
2327                         amdgpu_benchmark(adev, amdgpu_benchmarking);
2328                 else
2329                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2330         }
2331
2332         /* enable clockgating, etc. after ib tests, etc. since some blocks require
2333          * explicit gating rather than handling it automatically.
2334          */
2335         r = amdgpu_late_init(adev);
2336         if (r) {
2337                 dev_err(adev->dev, "amdgpu_late_init failed\n");
2338                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2339                 goto failed;
2340         }
2341
2342         return 0;
2343
2344 failed:
2345         amdgpu_vf_error_trans_all(adev);
2346         if (runtime)
2347                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2348         return r;
2349 }
2350
2351 /**
2352  * amdgpu_device_fini - tear down the driver
2353  *
2354  * @adev: amdgpu_device pointer
2355  *
2356  * Tear down the driver info (all asics).
2357  * Called at driver shutdown.
2358  */
2359 void amdgpu_device_fini(struct amdgpu_device *adev)
2360 {
2361         int r;
2362
2363         DRM_INFO("amdgpu: finishing device.\n");
2364         adev->shutdown = true;
2365         if (adev->mode_info.mode_config_initialized)
2366                 drm_crtc_force_disable_all(adev->ddev);
2367         /* evict vram memory */
2368         amdgpu_bo_evict_vram(adev);
2369         amdgpu_ib_pool_fini(adev);
2370         amdgpu_fw_reserve_vram_fini(adev);
2371         amdgpu_fence_driver_fini(adev);
2372         amdgpu_fbdev_fini(adev);
2373         r = amdgpu_fini(adev);
2374         if (adev->firmware.gpu_info_fw) {
2375                 release_firmware(adev->firmware.gpu_info_fw);
2376                 adev->firmware.gpu_info_fw = NULL;
2377         }
2378         adev->accel_working = false;
2379         cancel_delayed_work_sync(&adev->late_init_work);
2380         /* free i2c buses */
2381         amdgpu_i2c_fini(adev);
2382         amdgpu_atombios_fini(adev);
2383         kfree(adev->bios);
2384         adev->bios = NULL;
2385         if (!pci_is_thunderbolt_attached(adev->pdev))
2386                 vga_switcheroo_unregister_client(adev->pdev);
2387         if (adev->flags & AMD_IS_PX)
2388                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2389         vga_client_register(adev->pdev, NULL, NULL, NULL);
2390         if (adev->rio_mem)
2391                 pci_iounmap(adev->pdev, adev->rio_mem);
2392         adev->rio_mem = NULL;
2393         iounmap(adev->rmmio);
2394         adev->rmmio = NULL;
2395         amdgpu_doorbell_fini(adev);
2396         amdgpu_pm_sysfs_fini(adev);
2397         amdgpu_debugfs_regs_cleanup(adev);
2398 }
2399
2400
2401 /*
2402  * Suspend & resume.
2403  */
2404 /**
2405  * amdgpu_device_suspend - initiate device suspend
2406  *
2407  * @pdev: drm dev pointer
2408  * @state: suspend state
2409  *
2410  * Puts the hw in the suspend state (all asics).
2411  * Returns 0 for success or an error on failure.
2412  * Called at driver suspend.
2413  */
2414 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2415 {
2416         struct amdgpu_device *adev;
2417         struct drm_crtc *crtc;
2418         struct drm_connector *connector;
2419         int r;
2420
2421         if (dev == NULL || dev->dev_private == NULL) {
2422                 return -ENODEV;
2423         }
2424
2425         adev = dev->dev_private;
2426
2427         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2428                 return 0;
2429
2430         drm_kms_helper_poll_disable(dev);
2431
2432         /* turn off display hw */
2433         drm_modeset_lock_all(dev);
2434         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2435                 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2436         }
2437         drm_modeset_unlock_all(dev);
2438
2439         amdgpu_amdkfd_suspend(adev);
2440
2441         /* unpin the front buffers and cursors */
2442         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2443                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2444                 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
2445                 struct amdgpu_bo *robj;
2446
2447                 if (amdgpu_crtc->cursor_bo) {
2448                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2449                         r = amdgpu_bo_reserve(aobj, true);
2450                         if (r == 0) {
2451                                 amdgpu_bo_unpin(aobj);
2452                                 amdgpu_bo_unreserve(aobj);
2453                         }
2454                 }
2455
2456                 if (rfb == NULL || rfb->obj == NULL) {
2457                         continue;
2458                 }
2459                 robj = gem_to_amdgpu_bo(rfb->obj);
2460                 /* don't unpin kernel fb objects */
2461                 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2462                         r = amdgpu_bo_reserve(robj, true);
2463                         if (r == 0) {
2464                                 amdgpu_bo_unpin(robj);
2465                                 amdgpu_bo_unreserve(robj);
2466                         }
2467                 }
2468         }
2469         /* evict vram memory */
2470         amdgpu_bo_evict_vram(adev);
2471
2472         amdgpu_fence_driver_suspend(adev);
2473
2474         r = amdgpu_suspend(adev);
2475
2476         /* evict remaining vram memory
2477          * This second call to evict vram is to evict the gart page table
2478          * using the CPU.
2479          */
2480         amdgpu_bo_evict_vram(adev);
2481
2482         amdgpu_atombios_scratch_regs_save(adev);
2483         pci_save_state(dev->pdev);
2484         if (suspend) {
2485                 /* Shut down the device */
2486                 pci_disable_device(dev->pdev);
2487                 pci_set_power_state(dev->pdev, PCI_D3hot);
2488         } else {
2489                 r = amdgpu_asic_reset(adev);
2490                 if (r)
2491                         DRM_ERROR("amdgpu asic reset failed\n");
2492         }
2493
2494         if (fbcon) {
2495                 console_lock();
2496                 amdgpu_fbdev_set_suspend(adev, 1);
2497                 console_unlock();
2498         }
2499         return 0;
2500 }
2501
2502 /**
2503  * amdgpu_device_resume - initiate device resume
2504  *
2505  * @pdev: drm dev pointer
2506  *
2507  * Bring the hw back to operating state (all asics).
2508  * Returns 0 for success or an error on failure.
2509  * Called at driver resume.
2510  */
2511 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2512 {
2513         struct drm_connector *connector;
2514         struct amdgpu_device *adev = dev->dev_private;
2515         struct drm_crtc *crtc;
2516         int r = 0;
2517
2518         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2519                 return 0;
2520
2521         if (fbcon)
2522                 console_lock();
2523
2524         if (resume) {
2525                 pci_set_power_state(dev->pdev, PCI_D0);
2526                 pci_restore_state(dev->pdev);
2527                 r = pci_enable_device(dev->pdev);
2528                 if (r)
2529                         goto unlock;
2530         }
2531         amdgpu_atombios_scratch_regs_restore(adev);
2532
2533         /* post card */
2534         if (amdgpu_need_post(adev)) {
2535                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2536                 if (r)
2537                         DRM_ERROR("amdgpu asic init failed\n");
2538         }
2539
2540         r = amdgpu_resume(adev);
2541         if (r) {
2542                 DRM_ERROR("amdgpu_resume failed (%d).\n", r);
2543                 goto unlock;
2544         }
2545         amdgpu_fence_driver_resume(adev);
2546
2547         if (resume) {
2548                 r = amdgpu_ib_ring_tests(adev);
2549                 if (r)
2550                         DRM_ERROR("ib ring test failed (%d).\n", r);
2551         }
2552
2553         r = amdgpu_late_init(adev);
2554         if (r)
2555                 goto unlock;
2556
2557         /* pin cursors */
2558         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2559                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2560
2561                 if (amdgpu_crtc->cursor_bo) {
2562                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2563                         r = amdgpu_bo_reserve(aobj, true);
2564                         if (r == 0) {
2565                                 r = amdgpu_bo_pin(aobj,
2566                                                   AMDGPU_GEM_DOMAIN_VRAM,
2567                                                   &amdgpu_crtc->cursor_addr);
2568                                 if (r != 0)
2569                                         DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2570                                 amdgpu_bo_unreserve(aobj);
2571                         }
2572                 }
2573         }
2574         r = amdgpu_amdkfd_resume(adev);
2575         if (r)
2576                 return r;
2577
2578         /* blat the mode back in */
2579         if (fbcon) {
2580                 drm_helper_resume_force_mode(dev);
2581                 /* turn on display hw */
2582                 drm_modeset_lock_all(dev);
2583                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2584                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2585                 }
2586                 drm_modeset_unlock_all(dev);
2587         }
2588
2589         drm_kms_helper_poll_enable(dev);
2590
2591         /*
2592          * Most of the connector probing functions try to acquire runtime pm
2593          * refs to ensure that the GPU is powered on when connector polling is
2594          * performed. Since we're calling this from a runtime PM callback,
2595          * trying to acquire rpm refs will cause us to deadlock.
2596          *
2597          * Since we're guaranteed to be holding the rpm lock, it's safe to
2598          * temporarily disable the rpm helpers so this doesn't deadlock us.
2599          */
2600 #ifdef CONFIG_PM
2601         dev->dev->power.disable_depth++;
2602 #endif
2603         drm_helper_hpd_irq_event(dev);
2604 #ifdef CONFIG_PM
2605         dev->dev->power.disable_depth--;
2606 #endif
2607
2608         if (fbcon)
2609                 amdgpu_fbdev_set_suspend(adev, 0);
2610
2611 unlock:
2612         if (fbcon)
2613                 console_unlock();
2614
2615         return r;
2616 }
2617
2618 static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
2619 {
2620         int i;
2621         bool asic_hang = false;
2622
2623         if (amdgpu_sriov_vf(adev))
2624                 return true;
2625
2626         for (i = 0; i < adev->num_ip_blocks; i++) {
2627                 if (!adev->ip_blocks[i].status.valid)
2628                         continue;
2629                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2630                         adev->ip_blocks[i].status.hang =
2631                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2632                 if (adev->ip_blocks[i].status.hang) {
2633                         DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2634                         asic_hang = true;
2635                 }
2636         }
2637         return asic_hang;
2638 }
2639
2640 static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
2641 {
2642         int i, r = 0;
2643
2644         for (i = 0; i < adev->num_ip_blocks; i++) {
2645                 if (!adev->ip_blocks[i].status.valid)
2646                         continue;
2647                 if (adev->ip_blocks[i].status.hang &&
2648                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2649                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2650                         if (r)
2651                                 return r;
2652                 }
2653         }
2654
2655         return 0;
2656 }
2657
2658 static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
2659 {
2660         int i;
2661
2662         for (i = 0; i < adev->num_ip_blocks; i++) {
2663                 if (!adev->ip_blocks[i].status.valid)
2664                         continue;
2665                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2666                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2667                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2668                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
2669                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2670                         if (adev->ip_blocks[i].status.hang) {
2671                                 DRM_INFO("Some block need full reset!\n");
2672                                 return true;
2673                         }
2674                 }
2675         }
2676         return false;
2677 }
2678
2679 static int amdgpu_soft_reset(struct amdgpu_device *adev)
2680 {
2681         int i, r = 0;
2682
2683         for (i = 0; i < adev->num_ip_blocks; i++) {
2684                 if (!adev->ip_blocks[i].status.valid)
2685                         continue;
2686                 if (adev->ip_blocks[i].status.hang &&
2687                     adev->ip_blocks[i].version->funcs->soft_reset) {
2688                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2689                         if (r)
2690                                 return r;
2691                 }
2692         }
2693
2694         return 0;
2695 }
2696
2697 static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
2698 {
2699         int i, r = 0;
2700
2701         for (i = 0; i < adev->num_ip_blocks; i++) {
2702                 if (!adev->ip_blocks[i].status.valid)
2703                         continue;
2704                 if (adev->ip_blocks[i].status.hang &&
2705                     adev->ip_blocks[i].version->funcs->post_soft_reset)
2706                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2707                 if (r)
2708                         return r;
2709         }
2710
2711         return 0;
2712 }
2713
2714 bool amdgpu_need_backup(struct amdgpu_device *adev)
2715 {
2716         if (adev->flags & AMD_IS_APU)
2717                 return false;
2718
2719         return amdgpu_lockup_timeout > 0 ? true : false;
2720 }
2721
2722 static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
2723                                            struct amdgpu_ring *ring,
2724                                            struct amdgpu_bo *bo,
2725                                            struct dma_fence **fence)
2726 {
2727         uint32_t domain;
2728         int r;
2729
2730         if (!bo->shadow)
2731                 return 0;
2732
2733         r = amdgpu_bo_reserve(bo, true);
2734         if (r)
2735                 return r;
2736         domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2737         /* if bo has been evicted, then no need to recover */
2738         if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2739                 r = amdgpu_bo_validate(bo->shadow);
2740                 if (r) {
2741                         DRM_ERROR("bo validate failed!\n");
2742                         goto err;
2743                 }
2744
2745                 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2746                                                  NULL, fence, true);
2747                 if (r) {
2748                         DRM_ERROR("recover page table failed!\n");
2749                         goto err;
2750                 }
2751         }
2752 err:
2753         amdgpu_bo_unreserve(bo);
2754         return r;
2755 }
2756
2757 /**
2758  * amdgpu_sriov_gpu_reset - reset the asic
2759  *
2760  * @adev: amdgpu device pointer
2761  * @job: which job trigger hang
2762  *
2763  * Attempt the reset the GPU if it has hung (all asics).
2764  * for SRIOV case.
2765  * Returns 0 for success or an error on failure.
2766  */
2767 int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
2768 {
2769         int i, j, r = 0;
2770         int resched;
2771         struct amdgpu_bo *bo, *tmp;
2772         struct amdgpu_ring *ring;
2773         struct dma_fence *fence = NULL, *next = NULL;
2774
2775         mutex_lock(&adev->virt.lock_reset);
2776         atomic_inc(&adev->gpu_reset_counter);
2777         adev->in_sriov_reset = true;
2778
2779         /* block TTM */
2780         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2781
2782         /* we start from the ring trigger GPU hang */
2783         j = job ? job->ring->idx : 0;
2784
2785         /* block scheduler */
2786         for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
2787                 ring = adev->rings[i % AMDGPU_MAX_RINGS];
2788                 if (!ring || !ring->sched.thread)
2789                         continue;
2790
2791                 kthread_park(ring->sched.thread);
2792
2793                 if (job && j != i)
2794                         continue;
2795
2796                 /* here give the last chance to check if job removed from mirror-list
2797                  * since we already pay some time on kthread_park */
2798                 if (job && list_empty(&job->base.node)) {
2799                         kthread_unpark(ring->sched.thread);
2800                         goto give_up_reset;
2801                 }
2802
2803                 if (amd_sched_invalidate_job(&job->base, amdgpu_job_hang_limit))
2804                         amd_sched_job_kickout(&job->base);
2805
2806                 /* only do job_reset on the hang ring if @job not NULL */
2807                 amd_sched_hw_job_reset(&ring->sched);
2808
2809                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2810                 amdgpu_fence_driver_force_completion_ring(ring);
2811         }
2812
2813         /* request to take full control of GPU before re-initialization  */
2814         if (job)
2815                 amdgpu_virt_reset_gpu(adev);
2816         else
2817                 amdgpu_virt_request_full_gpu(adev, true);
2818
2819
2820         /* Resume IP prior to SMC */
2821         amdgpu_sriov_reinit_early(adev);
2822
2823         /* we need recover gart prior to run SMC/CP/SDMA resume */
2824         amdgpu_ttm_recover_gart(adev);
2825
2826         /* now we are okay to resume SMC/CP/SDMA */
2827         amdgpu_sriov_reinit_late(adev);
2828
2829         amdgpu_irq_gpu_reset_resume_helper(adev);
2830
2831         if (amdgpu_ib_ring_tests(adev))
2832                 dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
2833
2834         /* release full control of GPU after ib test */
2835         amdgpu_virt_release_full_gpu(adev, true);
2836
2837         DRM_INFO("recover vram bo from shadow\n");
2838
2839         ring = adev->mman.buffer_funcs_ring;
2840         mutex_lock(&adev->shadow_list_lock);
2841         list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2842                 next = NULL;
2843                 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2844                 if (fence) {
2845                         r = dma_fence_wait(fence, false);
2846                         if (r) {
2847                                 WARN(r, "recovery from shadow isn't completed\n");
2848                                 break;
2849                         }
2850                 }
2851
2852                 dma_fence_put(fence);
2853                 fence = next;
2854         }
2855         mutex_unlock(&adev->shadow_list_lock);
2856
2857         if (fence) {
2858                 r = dma_fence_wait(fence, false);
2859                 if (r)
2860                         WARN(r, "recovery from shadow isn't completed\n");
2861         }
2862         dma_fence_put(fence);
2863
2864         for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
2865                 ring = adev->rings[i % AMDGPU_MAX_RINGS];
2866                 if (!ring || !ring->sched.thread)
2867                         continue;
2868
2869                 if (job && j != i) {
2870                         kthread_unpark(ring->sched.thread);
2871                         continue;
2872                 }
2873
2874                 amd_sched_job_recovery(&ring->sched);
2875                 kthread_unpark(ring->sched.thread);
2876         }
2877
2878         drm_helper_resume_force_mode(adev->ddev);
2879 give_up_reset:
2880         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2881         if (r) {
2882                 /* bad news, how to tell it to userspace ? */
2883                 dev_info(adev->dev, "GPU reset failed\n");
2884         } else {
2885                 dev_info(adev->dev, "GPU reset successed!\n");
2886         }
2887
2888         adev->in_sriov_reset = false;
2889         mutex_unlock(&adev->virt.lock_reset);
2890         return r;
2891 }
2892
2893 /**
2894  * amdgpu_gpu_reset - reset the asic
2895  *
2896  * @adev: amdgpu device pointer
2897  *
2898  * Attempt the reset the GPU if it has hung (all asics).
2899  * Returns 0 for success or an error on failure.
2900  */
2901 int amdgpu_gpu_reset(struct amdgpu_device *adev)
2902 {
2903         int i, r;
2904         int resched;
2905         bool need_full_reset, vram_lost = false;
2906
2907         if (!amdgpu_check_soft_reset(adev)) {
2908                 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
2909                 return 0;
2910         }
2911
2912         atomic_inc(&adev->gpu_reset_counter);
2913
2914         /* block TTM */
2915         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2916
2917         /* block scheduler */
2918         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2919                 struct amdgpu_ring *ring = adev->rings[i];
2920
2921                 if (!ring || !ring->sched.thread)
2922                         continue;
2923                 kthread_park(ring->sched.thread);
2924                 amd_sched_hw_job_reset(&ring->sched);
2925         }
2926         /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2927         amdgpu_fence_driver_force_completion(adev);
2928
2929         need_full_reset = amdgpu_need_full_reset(adev);
2930
2931         if (!need_full_reset) {
2932                 amdgpu_pre_soft_reset(adev);
2933                 r = amdgpu_soft_reset(adev);
2934                 amdgpu_post_soft_reset(adev);
2935                 if (r || amdgpu_check_soft_reset(adev)) {
2936                         DRM_INFO("soft reset failed, will fallback to full reset!\n");
2937                         need_full_reset = true;
2938                 }
2939         }
2940
2941         if (need_full_reset) {
2942                 r = amdgpu_suspend(adev);
2943
2944 retry:
2945                 amdgpu_atombios_scratch_regs_save(adev);
2946                 r = amdgpu_asic_reset(adev);
2947                 amdgpu_atombios_scratch_regs_restore(adev);
2948                 /* post card */
2949                 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2950
2951                 if (!r) {
2952                         dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
2953                         r = amdgpu_resume_phase1(adev);
2954                         if (r)
2955                                 goto out;
2956                         vram_lost = amdgpu_check_vram_lost(adev);
2957                         if (vram_lost) {
2958                                 DRM_ERROR("VRAM is lost!\n");
2959                                 atomic_inc(&adev->vram_lost_counter);
2960                         }
2961                         r = amdgpu_ttm_recover_gart(adev);
2962                         if (r)
2963                                 goto out;
2964                         r = amdgpu_resume_phase2(adev);
2965                         if (r)
2966                                 goto out;
2967                         if (vram_lost)
2968                                 amdgpu_fill_reset_magic(adev);
2969                 }
2970         }
2971 out:
2972         if (!r) {
2973                 amdgpu_irq_gpu_reset_resume_helper(adev);
2974                 r = amdgpu_ib_ring_tests(adev);
2975                 if (r) {
2976                         dev_err(adev->dev, "ib ring test failed (%d).\n", r);
2977                         r = amdgpu_suspend(adev);
2978                         need_full_reset = true;
2979                         goto retry;
2980                 }
2981                 /**
2982                  * recovery vm page tables, since we cannot depend on VRAM is
2983                  * consistent after gpu full reset.
2984                  */
2985                 if (need_full_reset && amdgpu_need_backup(adev)) {
2986                         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2987                         struct amdgpu_bo *bo, *tmp;
2988                         struct dma_fence *fence = NULL, *next = NULL;
2989
2990                         DRM_INFO("recover vram bo from shadow\n");
2991                         mutex_lock(&adev->shadow_list_lock);
2992                         list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2993                                 next = NULL;
2994                                 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2995                                 if (fence) {
2996                                         r = dma_fence_wait(fence, false);
2997                                         if (r) {
2998                                                 WARN(r, "recovery from shadow isn't completed\n");
2999                                                 break;
3000                                         }
3001                                 }
3002
3003                                 dma_fence_put(fence);
3004                                 fence = next;
3005                         }
3006                         mutex_unlock(&adev->shadow_list_lock);
3007                         if (fence) {
3008                                 r = dma_fence_wait(fence, false);
3009                                 if (r)
3010                                         WARN(r, "recovery from shadow isn't completed\n");
3011                         }
3012                         dma_fence_put(fence);
3013                 }
3014                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3015                         struct amdgpu_ring *ring = adev->rings[i];
3016
3017                         if (!ring || !ring->sched.thread)
3018                                 continue;
3019
3020                         amd_sched_job_recovery(&ring->sched);
3021                         kthread_unpark(ring->sched.thread);
3022                 }
3023         } else {
3024                 dev_err(adev->dev, "asic resume failed (%d).\n", r);
3025                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3026                         if (adev->rings[i] && adev->rings[i]->sched.thread) {
3027                                 kthread_unpark(adev->rings[i]->sched.thread);
3028                         }
3029                 }
3030         }
3031
3032         drm_helper_resume_force_mode(adev->ddev);
3033
3034         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
3035         if (r) {
3036                 /* bad news, how to tell it to userspace ? */
3037                 dev_info(adev->dev, "GPU reset failed\n");
3038         }
3039         else {
3040                 dev_info(adev->dev, "GPU reset successed!\n");
3041         }
3042
3043         amdgpu_vf_error_trans_all(adev);
3044         return r;
3045 }
3046
3047 void amdgpu_get_pcie_info(struct amdgpu_device *adev)
3048 {
3049         u32 mask;
3050         int ret;
3051
3052         if (amdgpu_pcie_gen_cap)
3053                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
3054
3055         if (amdgpu_pcie_lane_cap)
3056                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
3057
3058         /* covers APUs as well */
3059         if (pci_is_root_bus(adev->pdev->bus)) {
3060                 if (adev->pm.pcie_gen_mask == 0)
3061                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3062                 if (adev->pm.pcie_mlw_mask == 0)
3063                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3064                 return;
3065         }
3066
3067         if (adev->pm.pcie_gen_mask == 0) {
3068                 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
3069                 if (!ret) {
3070                         adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3071                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3072                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
3073
3074                         if (mask & DRM_PCIE_SPEED_25)
3075                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
3076                         if (mask & DRM_PCIE_SPEED_50)
3077                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
3078                         if (mask & DRM_PCIE_SPEED_80)
3079                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
3080                 } else {
3081                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3082                 }
3083         }
3084         if (adev->pm.pcie_mlw_mask == 0) {
3085                 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
3086                 if (!ret) {
3087                         switch (mask) {
3088                         case 32:
3089                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
3090                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3091                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3092                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3093                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3094                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3095                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3096                                 break;
3097                         case 16:
3098                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3099                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3100                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3101                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3102                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3103                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3104                                 break;
3105                         case 12:
3106                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3107                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3108                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3109                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3110                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3111                                 break;
3112                         case 8:
3113                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3114                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3115                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3116                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3117                                 break;
3118                         case 4:
3119                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3120                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3121                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3122                                 break;
3123                         case 2:
3124                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3125                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3126                                 break;
3127                         case 1:
3128                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
3129                                 break;
3130                         default:
3131                                 break;
3132                         }
3133                 } else {
3134                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3135                 }
3136         }
3137 }
3138
3139 /*
3140  * Debugfs
3141  */
3142 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
3143                              const struct drm_info_list *files,
3144                              unsigned nfiles)
3145 {
3146         unsigned i;
3147
3148         for (i = 0; i < adev->debugfs_count; i++) {
3149                 if (adev->debugfs[i].files == files) {
3150                         /* Already registered */
3151                         return 0;
3152                 }
3153         }
3154
3155         i = adev->debugfs_count + 1;
3156         if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
3157                 DRM_ERROR("Reached maximum number of debugfs components.\n");
3158                 DRM_ERROR("Report so we increase "
3159                           "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
3160                 return -EINVAL;
3161         }
3162         adev->debugfs[adev->debugfs_count].files = files;
3163         adev->debugfs[adev->debugfs_count].num_files = nfiles;
3164         adev->debugfs_count = i;
3165 #if defined(CONFIG_DEBUG_FS)
3166         drm_debugfs_create_files(files, nfiles,
3167                                  adev->ddev->primary->debugfs_root,
3168                                  adev->ddev->primary);
3169 #endif
3170         return 0;
3171 }
3172
3173 #if defined(CONFIG_DEBUG_FS)
3174
3175 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
3176                                         size_t size, loff_t *pos)
3177 {
3178         struct amdgpu_device *adev = file_inode(f)->i_private;
3179         ssize_t result = 0;
3180         int r;
3181         bool pm_pg_lock, use_bank;
3182         unsigned instance_bank, sh_bank, se_bank;
3183
3184         if (size & 0x3 || *pos & 0x3)
3185                 return -EINVAL;
3186
3187         /* are we reading registers for which a PG lock is necessary? */
3188         pm_pg_lock = (*pos >> 23) & 1;
3189
3190         if (*pos & (1ULL << 62)) {
3191                 se_bank = (*pos >> 24) & 0x3FF;
3192                 sh_bank = (*pos >> 34) & 0x3FF;
3193                 instance_bank = (*pos >> 44) & 0x3FF;
3194
3195                 if (se_bank == 0x3FF)
3196                         se_bank = 0xFFFFFFFF;
3197                 if (sh_bank == 0x3FF)
3198                         sh_bank = 0xFFFFFFFF;
3199                 if (instance_bank == 0x3FF)
3200                         instance_bank = 0xFFFFFFFF;
3201                 use_bank = 1;
3202         } else {
3203                 use_bank = 0;
3204         }
3205
3206         *pos &= (1UL << 22) - 1;
3207
3208         if (use_bank) {
3209                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
3210                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
3211                         return -EINVAL;
3212                 mutex_lock(&adev->grbm_idx_mutex);
3213                 amdgpu_gfx_select_se_sh(adev, se_bank,
3214                                         sh_bank, instance_bank);
3215         }
3216
3217         if (pm_pg_lock)
3218                 mutex_lock(&adev->pm.mutex);
3219
3220         while (size) {
3221                 uint32_t value;
3222
3223                 if (*pos > adev->rmmio_size)
3224                         goto end;
3225
3226                 value = RREG32(*pos >> 2);
3227                 r = put_user(value, (uint32_t *)buf);
3228                 if (r) {
3229                         result = r;
3230                         goto end;
3231                 }
3232
3233                 result += 4;
3234                 buf += 4;
3235                 *pos += 4;
3236                 size -= 4;
3237         }
3238
3239 end:
3240         if (use_bank) {
3241                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3242                 mutex_unlock(&adev->grbm_idx_mutex);
3243         }
3244
3245         if (pm_pg_lock)
3246                 mutex_unlock(&adev->pm.mutex);
3247
3248         return result;
3249 }
3250
3251 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
3252                                          size_t size, loff_t *pos)
3253 {
3254         struct amdgpu_device *adev = file_inode(f)->i_private;
3255         ssize_t result = 0;
3256         int r;
3257         bool pm_pg_lock, use_bank;
3258         unsigned instance_bank, sh_bank, se_bank;
3259
3260         if (size & 0x3 || *pos & 0x3)
3261                 return -EINVAL;
3262
3263         /* are we reading registers for which a PG lock is necessary? */
3264         pm_pg_lock = (*pos >> 23) & 1;
3265
3266         if (*pos & (1ULL << 62)) {
3267                 se_bank = (*pos >> 24) & 0x3FF;
3268                 sh_bank = (*pos >> 34) & 0x3FF;
3269                 instance_bank = (*pos >> 44) & 0x3FF;
3270
3271                 if (se_bank == 0x3FF)
3272                         se_bank = 0xFFFFFFFF;
3273                 if (sh_bank == 0x3FF)
3274                         sh_bank = 0xFFFFFFFF;
3275                 if (instance_bank == 0x3FF)
3276                         instance_bank = 0xFFFFFFFF;
3277                 use_bank = 1;
3278         } else {
3279                 use_bank = 0;
3280         }
3281
3282         *pos &= (1UL << 22) - 1;
3283
3284         if (use_bank) {
3285                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
3286                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
3287                         return -EINVAL;
3288                 mutex_lock(&adev->grbm_idx_mutex);
3289                 amdgpu_gfx_select_se_sh(adev, se_bank,
3290                                         sh_bank, instance_bank);
3291         }
3292
3293         if (pm_pg_lock)
3294                 mutex_lock(&adev->pm.mutex);
3295
3296         while (size) {
3297                 uint32_t value;
3298
3299                 if (*pos > adev->rmmio_size)
3300                         return result;
3301
3302                 r = get_user(value, (uint32_t *)buf);
3303                 if (r)
3304                         return r;
3305
3306                 WREG32(*pos >> 2, value);
3307
3308                 result += 4;
3309                 buf += 4;
3310                 *pos += 4;
3311                 size -= 4;
3312         }
3313
3314         if (use_bank) {
3315                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3316                 mutex_unlock(&adev->grbm_idx_mutex);
3317         }
3318
3319         if (pm_pg_lock)
3320                 mutex_unlock(&adev->pm.mutex);
3321
3322         return result;
3323 }
3324
3325 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
3326                                         size_t size, loff_t *pos)
3327 {
3328         struct amdgpu_device *adev = file_inode(f)->i_private;
3329         ssize_t result = 0;
3330         int r;
3331
3332         if (size & 0x3 || *pos & 0x3)
3333                 return -EINVAL;
3334
3335         while (size) {
3336                 uint32_t value;
3337
3338                 value = RREG32_PCIE(*pos >> 2);
3339                 r = put_user(value, (uint32_t *)buf);
3340                 if (r)
3341                         return r;
3342
3343                 result += 4;
3344                 buf += 4;
3345                 *pos += 4;
3346                 size -= 4;
3347         }
3348
3349         return result;
3350 }
3351
3352 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
3353                                          size_t size, loff_t *pos)
3354 {
3355         struct amdgpu_device *adev = file_inode(f)->i_private;
3356         ssize_t result = 0;
3357         int r;
3358
3359         if (size & 0x3 || *pos & 0x3)
3360                 return -EINVAL;
3361
3362         while (size) {
3363                 uint32_t value;
3364
3365                 r = get_user(value, (uint32_t *)buf);
3366                 if (r)
3367                         return r;
3368
3369                 WREG32_PCIE(*pos >> 2, value);
3370
3371                 result += 4;
3372                 buf += 4;
3373                 *pos += 4;
3374                 size -= 4;
3375         }
3376
3377         return result;
3378 }
3379
3380 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
3381                                         size_t size, loff_t *pos)
3382 {
3383         struct amdgpu_device *adev = file_inode(f)->i_private;
3384         ssize_t result = 0;
3385         int r;
3386
3387         if (size & 0x3 || *pos & 0x3)
3388                 return -EINVAL;
3389
3390         while (size) {
3391                 uint32_t value;
3392
3393                 value = RREG32_DIDT(*pos >> 2);
3394                 r = put_user(value, (uint32_t *)buf);
3395                 if (r)
3396                         return r;
3397
3398                 result += 4;
3399                 buf += 4;
3400                 *pos += 4;
3401                 size -= 4;
3402         }
3403
3404         return result;
3405 }
3406
3407 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
3408                                          size_t size, loff_t *pos)
3409 {
3410         struct amdgpu_device *adev = file_inode(f)->i_private;
3411         ssize_t result = 0;
3412         int r;
3413
3414         if (size & 0x3 || *pos & 0x3)
3415                 return -EINVAL;
3416
3417         while (size) {
3418                 uint32_t value;
3419
3420                 r = get_user(value, (uint32_t *)buf);
3421                 if (r)
3422                         return r;
3423
3424                 WREG32_DIDT(*pos >> 2, value);
3425
3426                 result += 4;
3427                 buf += 4;
3428                 *pos += 4;
3429                 size -= 4;
3430         }
3431
3432         return result;
3433 }
3434
3435 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
3436                                         size_t size, loff_t *pos)
3437 {
3438         struct amdgpu_device *adev = file_inode(f)->i_private;
3439         ssize_t result = 0;
3440         int r;
3441
3442         if (size & 0x3 || *pos & 0x3)
3443                 return -EINVAL;
3444
3445         while (size) {
3446                 uint32_t value;
3447
3448                 value = RREG32_SMC(*pos);
3449                 r = put_user(value, (uint32_t *)buf);
3450                 if (r)
3451                         return r;
3452
3453                 result += 4;
3454                 buf += 4;
3455                 *pos += 4;
3456                 size -= 4;
3457         }
3458
3459         return result;
3460 }
3461
3462 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
3463                                          size_t size, loff_t *pos)
3464 {
3465         struct amdgpu_device *adev = file_inode(f)->i_private;
3466         ssize_t result = 0;
3467         int r;
3468
3469         if (size & 0x3 || *pos & 0x3)
3470                 return -EINVAL;
3471
3472         while (size) {
3473                 uint32_t value;
3474
3475                 r = get_user(value, (uint32_t *)buf);
3476                 if (r)
3477                         return r;
3478
3479                 WREG32_SMC(*pos, value);
3480
3481                 result += 4;
3482                 buf += 4;
3483                 *pos += 4;
3484                 size -= 4;
3485         }
3486
3487         return result;
3488 }
3489
3490 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
3491                                         size_t size, loff_t *pos)
3492 {
3493         struct amdgpu_device *adev = file_inode(f)->i_private;
3494         ssize_t result = 0;
3495         int r;
3496         uint32_t *config, no_regs = 0;
3497
3498         if (size & 0x3 || *pos & 0x3)
3499                 return -EINVAL;
3500
3501         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
3502         if (!config)
3503                 return -ENOMEM;
3504
3505         /* version, increment each time something is added */
3506         config[no_regs++] = 3;
3507         config[no_regs++] = adev->gfx.config.max_shader_engines;
3508         config[no_regs++] = adev->gfx.config.max_tile_pipes;
3509         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
3510         config[no_regs++] = adev->gfx.config.max_sh_per_se;
3511         config[no_regs++] = adev->gfx.config.max_backends_per_se;
3512         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
3513         config[no_regs++] = adev->gfx.config.max_gprs;
3514         config[no_regs++] = adev->gfx.config.max_gs_threads;
3515         config[no_regs++] = adev->gfx.config.max_hw_contexts;
3516         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
3517         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
3518         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
3519         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
3520         config[no_regs++] = adev->gfx.config.num_tile_pipes;
3521         config[no_regs++] = adev->gfx.config.backend_enable_mask;
3522         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
3523         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
3524         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
3525         config[no_regs++] = adev->gfx.config.num_gpus;
3526         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
3527         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
3528         config[no_regs++] = adev->gfx.config.gb_addr_config;
3529         config[no_regs++] = adev->gfx.config.num_rbs;
3530
3531         /* rev==1 */
3532         config[no_regs++] = adev->rev_id;
3533         config[no_regs++] = adev->pg_flags;
3534         config[no_regs++] = adev->cg_flags;
3535
3536         /* rev==2 */
3537         config[no_regs++] = adev->family;
3538         config[no_regs++] = adev->external_rev_id;
3539
3540         /* rev==3 */
3541         config[no_regs++] = adev->pdev->device;
3542         config[no_regs++] = adev->pdev->revision;
3543         config[no_regs++] = adev->pdev->subsystem_device;
3544         config[no_regs++] = adev->pdev->subsystem_vendor;
3545
3546         while (size && (*pos < no_regs * 4)) {
3547                 uint32_t value;
3548
3549                 value = config[*pos >> 2];
3550                 r = put_user(value, (uint32_t *)buf);
3551                 if (r) {
3552                         kfree(config);
3553                         return r;
3554                 }
3555
3556                 result += 4;
3557                 buf += 4;
3558                 *pos += 4;
3559                 size -= 4;
3560         }
3561
3562         kfree(config);
3563         return result;
3564 }
3565
3566 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
3567                                         size_t size, loff_t *pos)
3568 {
3569         struct amdgpu_device *adev = file_inode(f)->i_private;
3570         int idx, x, outsize, r, valuesize;
3571         uint32_t values[16];
3572
3573         if (size & 3 || *pos & 0x3)
3574                 return -EINVAL;
3575
3576         if (amdgpu_dpm == 0)
3577                 return -EINVAL;
3578
3579         /* convert offset to sensor number */
3580         idx = *pos >> 2;
3581
3582         valuesize = sizeof(values);
3583         if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
3584                 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
3585         else
3586                 return -EINVAL;
3587
3588         if (size > valuesize)
3589                 return -EINVAL;
3590
3591         outsize = 0;
3592         x = 0;
3593         if (!r) {
3594                 while (size) {
3595                         r = put_user(values[x++], (int32_t *)buf);
3596                         buf += 4;
3597                         size -= 4;
3598                         outsize += 4;
3599                 }
3600         }
3601
3602         return !r ? outsize : r;
3603 }
3604
3605 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
3606                                         size_t size, loff_t *pos)
3607 {
3608         struct amdgpu_device *adev = f->f_inode->i_private;
3609         int r, x;
3610         ssize_t result=0;
3611         uint32_t offset, se, sh, cu, wave, simd, data[32];
3612
3613         if (size & 3 || *pos & 3)
3614                 return -EINVAL;
3615
3616         /* decode offset */
3617         offset = (*pos & 0x7F);
3618         se = ((*pos >> 7) & 0xFF);
3619         sh = ((*pos >> 15) & 0xFF);
3620         cu = ((*pos >> 23) & 0xFF);
3621         wave = ((*pos >> 31) & 0xFF);
3622         simd = ((*pos >> 37) & 0xFF);
3623
3624         /* switch to the specific se/sh/cu */
3625         mutex_lock(&adev->grbm_idx_mutex);
3626         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3627
3628         x = 0;
3629         if (adev->gfx.funcs->read_wave_data)
3630                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
3631
3632         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3633         mutex_unlock(&adev->grbm_idx_mutex);
3634
3635         if (!x)
3636                 return -EINVAL;
3637
3638         while (size && (offset < x * 4)) {
3639                 uint32_t value;
3640
3641                 value = data[offset >> 2];
3642                 r = put_user(value, (uint32_t *)buf);
3643                 if (r)
3644                         return r;
3645
3646                 result += 4;
3647                 buf += 4;
3648                 offset += 4;
3649                 size -= 4;
3650         }
3651
3652         return result;
3653 }
3654
3655 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
3656                                         size_t size, loff_t *pos)
3657 {
3658         struct amdgpu_device *adev = f->f_inode->i_private;
3659         int r;
3660         ssize_t result = 0;
3661         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
3662
3663         if (size & 3 || *pos & 3)
3664                 return -EINVAL;
3665
3666         /* decode offset */
3667         offset = (*pos & 0xFFF);       /* in dwords */
3668         se = ((*pos >> 12) & 0xFF);
3669         sh = ((*pos >> 20) & 0xFF);
3670         cu = ((*pos >> 28) & 0xFF);
3671         wave = ((*pos >> 36) & 0xFF);
3672         simd = ((*pos >> 44) & 0xFF);
3673         thread = ((*pos >> 52) & 0xFF);
3674         bank = ((*pos >> 60) & 1);
3675
3676         data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
3677         if (!data)
3678                 return -ENOMEM;
3679
3680         /* switch to the specific se/sh/cu */
3681         mutex_lock(&adev->grbm_idx_mutex);
3682         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3683
3684         if (bank == 0) {
3685                 if (adev->gfx.funcs->read_wave_vgprs)
3686                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
3687         } else {
3688                 if (adev->gfx.funcs->read_wave_sgprs)
3689                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
3690         }
3691
3692         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3693         mutex_unlock(&adev->grbm_idx_mutex);
3694
3695         while (size) {
3696                 uint32_t value;
3697
3698                 value = data[offset++];
3699                 r = put_user(value, (uint32_t *)buf);
3700                 if (r) {
3701                         result = r;
3702                         goto err;
3703                 }
3704
3705                 result += 4;
3706                 buf += 4;
3707                 size -= 4;
3708         }
3709
3710 err:
3711         kfree(data);
3712         return result;
3713 }
3714
3715 static const struct file_operations amdgpu_debugfs_regs_fops = {
3716         .owner = THIS_MODULE,
3717         .read = amdgpu_debugfs_regs_read,
3718         .write = amdgpu_debugfs_regs_write,
3719         .llseek = default_llseek
3720 };
3721 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
3722         .owner = THIS_MODULE,
3723         .read = amdgpu_debugfs_regs_didt_read,
3724         .write = amdgpu_debugfs_regs_didt_write,
3725         .llseek = default_llseek
3726 };
3727 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
3728         .owner = THIS_MODULE,
3729         .read = amdgpu_debugfs_regs_pcie_read,
3730         .write = amdgpu_debugfs_regs_pcie_write,
3731         .llseek = default_llseek
3732 };
3733 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
3734         .owner = THIS_MODULE,
3735         .read = amdgpu_debugfs_regs_smc_read,
3736         .write = amdgpu_debugfs_regs_smc_write,
3737         .llseek = default_llseek
3738 };
3739
3740 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
3741         .owner = THIS_MODULE,
3742         .read = amdgpu_debugfs_gca_config_read,
3743         .llseek = default_llseek
3744 };
3745
3746 static const struct file_operations amdgpu_debugfs_sensors_fops = {
3747         .owner = THIS_MODULE,
3748         .read = amdgpu_debugfs_sensor_read,
3749         .llseek = default_llseek
3750 };
3751
3752 static const struct file_operations amdgpu_debugfs_wave_fops = {
3753         .owner = THIS_MODULE,
3754         .read = amdgpu_debugfs_wave_read,
3755         .llseek = default_llseek
3756 };
3757 static const struct file_operations amdgpu_debugfs_gpr_fops = {
3758         .owner = THIS_MODULE,
3759         .read = amdgpu_debugfs_gpr_read,
3760         .llseek = default_llseek
3761 };
3762
3763 static const struct file_operations *debugfs_regs[] = {
3764         &amdgpu_debugfs_regs_fops,
3765         &amdgpu_debugfs_regs_didt_fops,
3766         &amdgpu_debugfs_regs_pcie_fops,
3767         &amdgpu_debugfs_regs_smc_fops,
3768         &amdgpu_debugfs_gca_config_fops,
3769         &amdgpu_debugfs_sensors_fops,
3770         &amdgpu_debugfs_wave_fops,
3771         &amdgpu_debugfs_gpr_fops,
3772 };
3773
3774 static const char *debugfs_regs_names[] = {
3775         "amdgpu_regs",
3776         "amdgpu_regs_didt",
3777         "amdgpu_regs_pcie",
3778         "amdgpu_regs_smc",
3779         "amdgpu_gca_config",
3780         "amdgpu_sensors",
3781         "amdgpu_wave",
3782         "amdgpu_gpr",
3783 };
3784
3785 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3786 {
3787         struct drm_minor *minor = adev->ddev->primary;
3788         struct dentry *ent, *root = minor->debugfs_root;
3789         unsigned i, j;
3790
3791         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3792                 ent = debugfs_create_file(debugfs_regs_names[i],
3793                                           S_IFREG | S_IRUGO, root,
3794                                           adev, debugfs_regs[i]);
3795                 if (IS_ERR(ent)) {
3796                         for (j = 0; j < i; j++) {
3797                                 debugfs_remove(adev->debugfs_regs[i]);
3798                                 adev->debugfs_regs[i] = NULL;
3799                         }
3800                         return PTR_ERR(ent);
3801                 }
3802
3803                 if (!i)
3804                         i_size_write(ent->d_inode, adev->rmmio_size);
3805                 adev->debugfs_regs[i] = ent;
3806         }
3807
3808         return 0;
3809 }
3810
3811 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
3812 {
3813         unsigned i;
3814
3815         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3816                 if (adev->debugfs_regs[i]) {
3817                         debugfs_remove(adev->debugfs_regs[i]);
3818                         adev->debugfs_regs[i] = NULL;
3819                 }
3820         }
3821 }
3822
3823 static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
3824 {
3825         struct drm_info_node *node = (struct drm_info_node *) m->private;
3826         struct drm_device *dev = node->minor->dev;
3827         struct amdgpu_device *adev = dev->dev_private;
3828         int r = 0, i;
3829
3830         /* hold on the scheduler */
3831         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
3832                 struct amdgpu_ring *ring = adev->rings[i];
3833
3834                 if (!ring || !ring->sched.thread)
3835                         continue;
3836                 kthread_park(ring->sched.thread);
3837         }
3838
3839         seq_printf(m, "run ib test:\n");
3840         r = amdgpu_ib_ring_tests(adev);
3841         if (r)
3842                 seq_printf(m, "ib ring tests failed (%d).\n", r);
3843         else
3844                 seq_printf(m, "ib ring tests passed.\n");
3845
3846         /* go on the scheduler */
3847         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
3848                 struct amdgpu_ring *ring = adev->rings[i];
3849
3850                 if (!ring || !ring->sched.thread)
3851                         continue;
3852                 kthread_unpark(ring->sched.thread);
3853         }
3854
3855         return 0;
3856 }
3857
3858 static const struct drm_info_list amdgpu_debugfs_test_ib_ring_list[] = {
3859         {"amdgpu_test_ib", &amdgpu_debugfs_test_ib}
3860 };
3861
3862 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
3863 {
3864         return amdgpu_debugfs_add_files(adev,
3865                                         amdgpu_debugfs_test_ib_ring_list, 1);
3866 }
3867
3868 int amdgpu_debugfs_init(struct drm_minor *minor)
3869 {
3870         return 0;
3871 }
3872
3873 static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
3874 {
3875         struct drm_info_node *node = (struct drm_info_node *) m->private;
3876         struct drm_device *dev = node->minor->dev;
3877         struct amdgpu_device *adev = dev->dev_private;
3878
3879         seq_write(m, adev->bios, adev->bios_size);
3880         return 0;
3881 }
3882
3883 static const struct drm_info_list amdgpu_vbios_dump_list[] = {
3884                 {"amdgpu_vbios",
3885                  amdgpu_debugfs_get_vbios_dump,
3886                  0, NULL},
3887 };
3888
3889 static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
3890 {
3891         return amdgpu_debugfs_add_files(adev,
3892                                         amdgpu_vbios_dump_list, 1);
3893 }
3894 #else
3895 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
3896 {
3897         return 0;
3898 }
3899 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3900 {
3901         return 0;
3902 }
3903 static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
3904 {
3905         return 0;
3906 }
3907 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
3908 #endif