drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/kthread.h>
  29 #include <linux/console.h>
  30 #include <linux/slab.h>
  31 #include <linux/debugfs.h>
  32 #include <drm/drmP.h>
  33 #include <drm/drm_crtc_helper.h>
  34 #include <drm/drm_atomic_helper.h>
  35 #include <drm/amdgpu_drm.h>
  36 #include <linux/vgaarb.h>
  37 #include <linux/vga_switcheroo.h>
  38 #include <linux/efi.h>
  39 #include "amdgpu.h"
  40 #include "amdgpu_trace.h"
  41 #include "amdgpu_i2c.h"
  42 #include "atom.h"
  43 #include "amdgpu_atombios.h"
  44 #include "amdgpu_atomfirmware.h"
  45 #include "amd_pcie.h"
  46 #ifdef CONFIG_DRM_AMDGPU_SI
  47 #include "si.h"
  48 #endif
  49 #ifdef CONFIG_DRM_AMDGPU_CIK
  50 #include "cik.h"
  51 #endif
  52 #include "vi.h"
  53 #include "soc15.h"
  54 #include "bif/bif_4_1_d.h"
  55 #include <linux/pci.h>
  56 #include <linux/firmware.h>
  57 #include "amdgpu_vf_error.h"
  58
  59 #include "amdgpu_amdkfd.h"
  60 #include "amdgpu_pm.h"
  61
  62 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  63 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  64
  65 #define AMDGPU_RESUME_MS                2000
  66
  67 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
  68 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
  69 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev);
  70 static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev);
  71
  72 static const char *amdgpu_asic_name[] = {
  73         "TAHITI",
  74         "PITCAIRN",
  75         "VERDE",
  76         "OLAND",
  77         "HAINAN",
  78         "BONAIRE",
  79         "KAVERI",
  80         "KABINI",
  81         "HAWAII",
  82         "MULLINS",
  83         "TOPAZ",
  84         "TONGA",
  85         "FIJI",
  86         "CARRIZO",
  87         "STONEY",
  88         "POLARIS10",
  89         "POLARIS11",
  90         "POLARIS12",
  91         "VEGA10",
  92         "RAVEN",
  93         "LAST",
  94 };
  95
  96 bool amdgpu_device_is_px(struct drm_device *dev)
  97 {
  98         struct amdgpu_device *adev = dev->dev_private;
  99
 100         if (adev->flags & AMD_IS_PX)
 101                 return true;
 102         return false;
 103 }
 104
 105 /*
 106  * MMIO register access helper functions.
 107  */
 108 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
 109                         uint32_t acc_flags)
 110 {
 111         uint32_t ret;
 112
 113         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
 114                 return amdgpu_virt_kiq_rreg(adev, reg);
 115
 116         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 117                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 118         else {
 119                 unsigned long flags;
 120
 121                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 122                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 123                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 124                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 125         }
 126         trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
 127         return ret;
 128 }
 129
 130 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 131                     uint32_t acc_flags)
 132 {
 133         trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 134
 135         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
 136                 adev->last_mm_index = v;
 137         }
 138
 139         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
 140                 return amdgpu_virt_kiq_wreg(adev, reg, v);
 141
 142         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 143                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 144         else {
 145                 unsigned long flags;
 146
 147                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 148                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 149                 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 150                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 151         }
 152
 153         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
 154                 udelay(500);
 155         }
 156 }
 157
 158 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
 159 {
 160         if ((reg * 4) < adev->rio_mem_size)
 161                 return ioread32(adev->rio_mem + (reg * 4));
 162         else {
 163                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 164                 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
 165         }
 166 }
 167
 168 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 169 {
 170         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
 171                 adev->last_mm_index = v;
 172         }
 173
 174         if ((reg * 4) < adev->rio_mem_size)
 175                 iowrite32(v, adev->rio_mem + (reg * 4));
 176         else {
 177                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 178                 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
 179         }
 180
 181         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
 182                 udelay(500);
 183         }
 184 }
 185
 186 /**
 187  * amdgpu_mm_rdoorbell - read a doorbell dword
 188  *
 189  * @adev: amdgpu_device pointer
 190  * @index: doorbell index
 191  *
 192  * Returns the value in the doorbell aperture at the
 193  * requested doorbell index (CIK).
 194  */
 195 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 196 {
 197         if (index < adev->doorbell.num_doorbells) {
 198                 return readl(adev->doorbell.ptr + index);
 199         } else {
 200                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 201                 return 0;
 202         }
 203 }
 204
 205 /**
 206  * amdgpu_mm_wdoorbell - write a doorbell dword
 207  *
 208  * @adev: amdgpu_device pointer
 209  * @index: doorbell index
 210  * @v: value to write
 211  *
 212  * Writes @v to the doorbell aperture at the
 213  * requested doorbell index (CIK).
 214  */
 215 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 216 {
 217         if (index < adev->doorbell.num_doorbells) {
 218                 writel(v, adev->doorbell.ptr + index);
 219         } else {
 220                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 221         }
 222 }
 223
 224 /**
 225  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 226  *
 227  * @adev: amdgpu_device pointer
 228  * @index: doorbell index
 229  *
 230  * Returns the value in the doorbell aperture at the
 231  * requested doorbell index (VEGA10+).
 232  */
 233 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 234 {
 235         if (index < adev->doorbell.num_doorbells) {
 236                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 237         } else {
 238                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 239                 return 0;
 240         }
 241 }
 242
 243 /**
 244  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 245  *
 246  * @adev: amdgpu_device pointer
 247  * @index: doorbell index
 248  * @v: value to write
 249  *
 250  * Writes @v to the doorbell aperture at the
 251  * requested doorbell index (VEGA10+).
 252  */
 253 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 254 {
 255         if (index < adev->doorbell.num_doorbells) {
 256                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 257         } else {
 258                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 259         }
 260 }
 261
 262 /**
 263  * amdgpu_invalid_rreg - dummy reg read function
 264  *
 265  * @adev: amdgpu device pointer
 266  * @reg: offset of register
 267  *
 268  * Dummy register read function.  Used for register blocks
 269  * that certain asics don't have (all asics).
 270  * Returns the value in the register.
 271  */
 272 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 273 {
 274         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 275         BUG();
 276         return 0;
 277 }
 278
 279 /**
 280  * amdgpu_invalid_wreg - dummy reg write function
 281  *
 282  * @adev: amdgpu device pointer
 283  * @reg: offset of register
 284  * @v: value to write to the register
 285  *
 286  * Dummy register read function.  Used for register blocks
 287  * that certain asics don't have (all asics).
 288  */
 289 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 290 {
 291         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 292                   reg, v);
 293         BUG();
 294 }
 295
 296 /**
 297  * amdgpu_block_invalid_rreg - dummy reg read function
 298  *
 299  * @adev: amdgpu device pointer
 300  * @block: offset of instance
 301  * @reg: offset of register
 302  *
 303  * Dummy register read function.  Used for register blocks
 304  * that certain asics don't have (all asics).
 305  * Returns the value in the register.
 306  */
 307 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 308                                           uint32_t block, uint32_t reg)
 309 {
 310         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 311                   reg, block);
 312         BUG();
 313         return 0;
 314 }
 315
 316 /**
 317  * amdgpu_block_invalid_wreg - dummy reg write function
 318  *
 319  * @adev: amdgpu device pointer
 320  * @block: offset of instance
 321  * @reg: offset of register
 322  * @v: value to write to the register
 323  *
 324  * Dummy register read function.  Used for register blocks
 325  * that certain asics don't have (all asics).
 326  */
 327 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 328                                       uint32_t block,
 329                                       uint32_t reg, uint32_t v)
 330 {
 331         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 332                   reg, block, v);
 333         BUG();
 334 }
 335
 336 static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
 337 {
 338         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 339                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 340                                        &adev->vram_scratch.robj,
 341                                        &adev->vram_scratch.gpu_addr,
 342                                        (void **)&adev->vram_scratch.ptr);
 343 }
 344
 345 static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
 346 {
 347         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 348 }
 349
 350 /**
 351  * amdgpu_program_register_sequence - program an array of registers.
 352  *
 353  * @adev: amdgpu_device pointer
 354  * @registers: pointer to the register array
 355  * @array_size: size of the register array
 356  *
 357  * Programs an array or registers with and and or masks.
 358  * This is a helper for setting golden registers.
 359  */
 360 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
 361                                       const u32 *registers,
 362                                       const u32 array_size)
 363 {
 364         u32 tmp, reg, and_mask, or_mask;
 365         int i;
 366
 367         if (array_size % 3)
 368                 return;
 369
 370         for (i = 0; i < array_size; i +=3) {
 371                 reg = registers[i + 0];
 372                 and_mask = registers[i + 1];
 373                 or_mask = registers[i + 2];
 374
 375                 if (and_mask == 0xffffffff) {
 376                         tmp = or_mask;
 377                 } else {
 378                         tmp = RREG32(reg);
 379                         tmp &= ~and_mask;
 380                         tmp |= or_mask;
 381                 }
 382                 WREG32(reg, tmp);
 383         }
 384 }
 385
 386 void amdgpu_pci_config_reset(struct amdgpu_device *adev)
 387 {
 388         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 389 }
 390
 391 /*
 392  * GPU doorbell aperture helpers function.
 393  */
 394 /**
 395  * amdgpu_doorbell_init - Init doorbell driver information.
 396  *
 397  * @adev: amdgpu_device pointer
 398  *
 399  * Init doorbell driver information (CIK)
 400  * Returns 0 on success, error on failure.
 401  */
 402 static int amdgpu_doorbell_init(struct amdgpu_device *adev)
 403 {
 404         /* No doorbell on SI hardware generation */
 405         if (adev->asic_type < CHIP_BONAIRE) {
 406                 adev->doorbell.base = 0;
 407                 adev->doorbell.size = 0;
 408                 adev->doorbell.num_doorbells = 0;
 409                 adev->doorbell.ptr = NULL;
 410                 return 0;
 411         }
 412
 413         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
 414                 return -EINVAL;
 415
 416         /* doorbell bar mapping */
 417         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 418         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 419
 420         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 421                                              AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
 422         if (adev->doorbell.num_doorbells == 0)
 423                 return -EINVAL;
 424
 425         adev->doorbell.ptr = ioremap(adev->doorbell.base,
 426                                      adev->doorbell.num_doorbells *
 427                                      sizeof(u32));
 428         if (adev->doorbell.ptr == NULL)
 429                 return -ENOMEM;
 430
 431         return 0;
 432 }
 433
 434 /**
 435  * amdgpu_doorbell_fini - Tear down doorbell driver information.
 436  *
 437  * @adev: amdgpu_device pointer
 438  *
 439  * Tear down doorbell driver information (CIK)
 440  */
 441 static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
 442 {
 443         iounmap(adev->doorbell.ptr);
 444         adev->doorbell.ptr = NULL;
 445 }
 446
 447 /**
 448  * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
 449  *                                setup amdkfd
 450  *
 451  * @adev: amdgpu_device pointer
 452  * @aperture_base: output returning doorbell aperture base physical address
 453  * @aperture_size: output returning doorbell aperture size in bytes
 454  * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
 455  *
 456  * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
 457  * takes doorbells required for its own rings and reports the setup to amdkfd.
 458  * amdgpu reserved doorbells are at the start of the doorbell aperture.
 459  */
 460 void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 461                                 phys_addr_t *aperture_base,
 462                                 size_t *aperture_size,
 463                                 size_t *start_offset)
 464 {
 465         /*
 466          * The first num_doorbells are used by amdgpu.
 467          * amdkfd takes whatever's left in the aperture.
 468          */
 469         if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
 470                 *aperture_base = adev->doorbell.base;
 471                 *aperture_size = adev->doorbell.size;
 472                 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
 473         } else {
 474                 *aperture_base = 0;
 475                 *aperture_size = 0;
 476                 *start_offset = 0;
 477         }
 478 }
 479
 480 /*
 481  * amdgpu_wb_*()
 482  * Writeback is the method by which the GPU updates special pages in memory
 483  * with the status of certain GPU events (fences, ring pointers,etc.).
 484  */
 485
 486 /**
 487  * amdgpu_wb_fini - Disable Writeback and free memory
 488  *
 489  * @adev: amdgpu_device pointer
 490  *
 491  * Disables Writeback and frees the Writeback memory (all asics).
 492  * Used at driver shutdown.
 493  */
 494 static void amdgpu_wb_fini(struct amdgpu_device *adev)
 495 {
 496         if (adev->wb.wb_obj) {
 497                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
 498                                       &adev->wb.gpu_addr,
 499                                       (void **)&adev->wb.wb);
 500                 adev->wb.wb_obj = NULL;
 501         }
 502 }
 503
 504 /**
 505  * amdgpu_wb_init- Init Writeback driver info and allocate memory
 506  *
 507  * @adev: amdgpu_device pointer
 508  *
 509  * Initializes writeback and allocates writeback memory (all asics).
 510  * Used at driver startup.
 511  * Returns 0 on success or an -error on failure.
 512  */
 513 static int amdgpu_wb_init(struct amdgpu_device *adev)
 514 {
 515         int r;
 516
 517         if (adev->wb.wb_obj == NULL) {
 518                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
 519                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
 520                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 521                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
 522                                             (void **)&adev->wb.wb);
 523                 if (r) {
 524                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
 525                         return r;
 526                 }
 527
 528                 adev->wb.num_wb = AMDGPU_MAX_WB;
 529                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
 530
 531                 /* clear wb memory */
 532                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t));
 533         }
 534
 535         return 0;
 536 }
 537
 538 /**
 539  * amdgpu_wb_get - Allocate a wb entry
 540  *
 541  * @adev: amdgpu_device pointer
 542  * @wb: wb index
 543  *
 544  * Allocate a wb slot for use by the driver (all asics).
 545  * Returns 0 on success or -EINVAL on failure.
 546  */
 547 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
 548 {
 549         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 550
 551         if (offset < adev->wb.num_wb) {
 552                 __set_bit(offset, adev->wb.used);
 553                 *wb = offset << 3; /* convert to dw offset */
 554                 return 0;
 555         } else {
 556                 return -EINVAL;
 557         }
 558 }
 559
 560 /**
 561  * amdgpu_wb_free - Free a wb entry
 562  *
 563  * @adev: amdgpu_device pointer
 564  * @wb: wb index
 565  *
 566  * Free a wb slot allocated for use by the driver (all asics)
 567  */
 568 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
 569 {
 570         if (wb < adev->wb.num_wb)
 571                 __clear_bit(wb >> 3, adev->wb.used);
 572 }
 573
 574 /**
 575  * amdgpu_vram_location - try to find VRAM location
 576  * @adev: amdgpu device structure holding all necessary informations
 577  * @mc: memory controller structure holding memory informations
 578  * @base: base address at which to put VRAM
 579  *
 580  * Function will try to place VRAM at base address provided
 581  * as parameter.
 582  */
 583 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
 584 {
 585         uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
 586
 587         mc->vram_start = base;
 588         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 589         if (limit && limit < mc->real_vram_size)
 590                 mc->real_vram_size = limit;
 591         dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 592                         mc->mc_vram_size >> 20, mc->vram_start,
 593                         mc->vram_end, mc->real_vram_size >> 20);
 594 }
 595
 596 /**
 597  * amdgpu_gart_location - try to find GTT location
 598  * @adev: amdgpu device structure holding all necessary informations
 599  * @mc: memory controller structure holding memory informations
 600  *
 601  * Function will place try to place GTT before or after VRAM.
 602  *
 603  * If GTT size is bigger than space left then we ajust GTT size.
 604  * Thus function will never fails.
 605  *
 606  * FIXME: when reducing GTT size align new size on power of 2.
 607  */
 608 void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
 609 {
 610         u64 size_af, size_bf;
 611
 612         size_af = adev->mc.mc_mask - mc->vram_end;
 613         size_bf = mc->vram_start;
 614         if (size_bf > size_af) {
 615                 if (mc->gart_size > size_bf) {
 616                         dev_warn(adev->dev, "limiting GTT\n");
 617                         mc->gart_size = size_bf;
 618                 }
 619                 mc->gart_start = 0;
 620         } else {
 621                 if (mc->gart_size > size_af) {
 622                         dev_warn(adev->dev, "limiting GTT\n");
 623                         mc->gart_size = size_af;
 624                 }
 625                 /* VCE doesn't like it when BOs cross a 4GB segment, so align
 626                  * the GART base on a 4GB boundary as well.
 627                  */
 628                 mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL);
 629         }
 630         mc->gart_end = mc->gart_start + mc->gart_size - 1;
 631         dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
 632                         mc->gart_size >> 20, mc->gart_start, mc->gart_end);
 633 }
 634
 635 /*
 636  * Firmware Reservation functions
 637  */
 638 /**
 639  * amdgpu_fw_reserve_vram_fini - free fw reserved vram
 640  *
 641  * @adev: amdgpu_device pointer
 642  *
 643  * free fw reserved vram if it has been reserved.
 644  */
 645 void amdgpu_fw_reserve_vram_fini(struct amdgpu_device *adev)
 646 {
 647         amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
 648                 NULL, &adev->fw_vram_usage.va);
 649 }
 650
 651 /**
 652  * amdgpu_fw_reserve_vram_init - create bo vram reservation from fw
 653  *
 654  * @adev: amdgpu_device pointer
 655  *
 656  * create bo vram reservation from fw.
 657  */
 658 int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
 659 {
 660         struct ttm_operation_ctx ctx = { false, false };
 661         int r = 0;
 662         int i;
 663         u64 vram_size = adev->mc.visible_vram_size;
 664         u64 offset = adev->fw_vram_usage.start_offset;
 665         u64 size = adev->fw_vram_usage.size;
 666         struct amdgpu_bo *bo;
 667
 668         adev->fw_vram_usage.va = NULL;
 669         adev->fw_vram_usage.reserved_bo = NULL;
 670
 671         if (adev->fw_vram_usage.size > 0 &&
 672                 adev->fw_vram_usage.size <= vram_size) {
 673
 674                 r = amdgpu_bo_create(adev, adev->fw_vram_usage.size,
 675                         PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
 676                         AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 677                         AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, NULL, NULL, 0,
 678                         &adev->fw_vram_usage.reserved_bo);
 679                 if (r)
 680                         goto error_create;
 681
 682                 r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
 683                 if (r)
 684                         goto error_reserve;
 685
 686                 /* remove the original mem node and create a new one at the
 687                  * request position
 688                  */
 689                 bo = adev->fw_vram_usage.reserved_bo;
 690                 offset = ALIGN(offset, PAGE_SIZE);
 691                 for (i = 0; i < bo->placement.num_placement; ++i) {
 692                         bo->placements[i].fpfn = offset >> PAGE_SHIFT;
 693                         bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
 694                 }
 695
 696                 ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem);
 697                 r = ttm_bo_mem_space(&bo->tbo, &bo->placement,
 698                                      &bo->tbo.mem, &ctx);
 699                 if (r)
 700                         goto error_pin;
 701
 702                 r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
 703                         AMDGPU_GEM_DOMAIN_VRAM,
 704                         adev->fw_vram_usage.start_offset,
 705                         (adev->fw_vram_usage.start_offset +
 706                         adev->fw_vram_usage.size), NULL);
 707                 if (r)
 708                         goto error_pin;
 709                 r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
 710                         &adev->fw_vram_usage.va);
 711                 if (r)
 712                         goto error_kmap;
 713
 714                 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
 715         }
 716         return r;
 717
 718 error_kmap:
 719         amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
 720 error_pin:
 721         amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
 722 error_reserve:
 723         amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
 724 error_create:
 725         adev->fw_vram_usage.va = NULL;
 726         adev->fw_vram_usage.reserved_bo = NULL;
 727         return r;
 728 }
 729
 730 /**
 731  * amdgpu_device_resize_fb_bar - try to resize FB BAR
 732  *
 733  * @adev: amdgpu_device pointer
 734  *
 735  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
 736  * to fail, but if any of the BARs is not accessible after the size we abort
 737  * driver loading by returning -ENODEV.
 738  */
 739 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
 740 {
 741         u64 space_needed = roundup_pow_of_two(adev->mc.real_vram_size);
 742         u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
 743         struct pci_bus *root;
 744         struct resource *res;
 745         unsigned i;
 746         u16 cmd;
 747         int r;
 748
 749         /* Bypass for VF */
 750         if (amdgpu_sriov_vf(adev))
 751                 return 0;
 752
 753         /* Check if the root BUS has 64bit memory resources */
 754         root = adev->pdev->bus;
 755         while (root->parent)
 756                 root = root->parent;
 757
 758         pci_bus_for_each_resource(root, res, i) {
 759                 if (res && res->flags & IORESOURCE_MEM_64 &&
 760                     res->start > 0x100000000ull)
 761                         break;
 762         }
 763
 764         /* Trying to resize is pointless without a root hub window above 4GB */
 765         if (!res)
 766                 return 0;
 767
 768         /* Disable memory decoding while we change the BAR addresses and size */
 769         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
 770         pci_write_config_word(adev->pdev, PCI_COMMAND,
 771                               cmd & ~PCI_COMMAND_MEMORY);
 772
 773         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
 774         amdgpu_doorbell_fini(adev);
 775         if (adev->asic_type >= CHIP_BONAIRE)
 776                 pci_release_resource(adev->pdev, 2);
 777
 778         pci_release_resource(adev->pdev, 0);
 779
 780         r = pci_resize_resource(adev->pdev, 0, rbar_size);
 781         if (r == -ENOSPC)
 782                 DRM_INFO("Not enough PCI address space for a large BAR.");
 783         else if (r && r != -ENOTSUPP)
 784                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
 785
 786         pci_assign_unassigned_bus_resources(adev->pdev->bus);
 787
 788         /* When the doorbell or fb BAR isn't available we have no chance of
 789          * using the device.
 790          */
 791         r = amdgpu_doorbell_init(adev);
 792         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
 793                 return -ENODEV;
 794
 795         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
 796
 797         return 0;
 798 }
 799
 800 /*
 801  * GPU helpers function.
 802  */
 803 /**
 804  * amdgpu_need_post - check if the hw need post or not
 805  *
 806  * @adev: amdgpu_device pointer
 807  *
 808  * Check if the asic has been initialized (all asics) at driver startup
 809  * or post is needed if  hw reset is performed.
 810  * Returns true if need or false if not.
 811  */
 812 bool amdgpu_need_post(struct amdgpu_device *adev)
 813 {
 814         uint32_t reg;
 815
 816         if (amdgpu_sriov_vf(adev))
 817                 return false;
 818
 819         if (amdgpu_passthrough(adev)) {
 820                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
 821                  * some old smc fw still need driver do vPost otherwise gpu hang, while
 822                  * those smc fw version above 22.15 doesn't have this flaw, so we force
 823                  * vpost executed for smc version below 22.15
 824                  */
 825                 if (adev->asic_type == CHIP_FIJI) {
 826                         int err;
 827                         uint32_t fw_ver;
 828                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
 829                         /* force vPost if error occured */
 830                         if (err)
 831                                 return true;
 832
 833                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
 834                         if (fw_ver < 0x00160e00)
 835                                 return true;
 836                 }
 837         }
 838
 839         if (adev->has_hw_reset) {
 840                 adev->has_hw_reset = false;
 841                 return true;
 842         }
 843
 844         /* bios scratch used on CIK+ */
 845         if (adev->asic_type >= CHIP_BONAIRE)
 846                 return amdgpu_atombios_scratch_need_asic_init(adev);
 847
 848         /* check MEM_SIZE for older asics */
 849         reg = amdgpu_asic_get_config_memsize(adev);
 850
 851         if ((reg != 0) && (reg != 0xffffffff))
 852                 return false;
 853
 854         return true;
 855 }
 856
 857 /**
 858  * amdgpu_dummy_page_init - init dummy page used by the driver
 859  *
 860  * @adev: amdgpu_device pointer
 861  *
 862  * Allocate the dummy page used by the driver (all asics).
 863  * This dummy page is used by the driver as a filler for gart entries
 864  * when pages are taken out of the GART
 865  * Returns 0 on sucess, -ENOMEM on failure.
 866  */
 867 int amdgpu_dummy_page_init(struct amdgpu_device *adev)
 868 {
 869         if (adev->dummy_page.page)
 870                 return 0;
 871         adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
 872         if (adev->dummy_page.page == NULL)
 873                 return -ENOMEM;
 874         adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
 875                                         0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 876         if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
 877                 dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
 878                 __free_page(adev->dummy_page.page);
 879                 adev->dummy_page.page = NULL;
 880                 return -ENOMEM;
 881         }
 882         return 0;
 883 }
 884
 885 /**
 886  * amdgpu_dummy_page_fini - free dummy page used by the driver
 887  *
 888  * @adev: amdgpu_device pointer
 889  *
 890  * Frees the dummy page used by the driver (all asics).
 891  */
 892 void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
 893 {
 894         if (adev->dummy_page.page == NULL)
 895                 return;
 896         pci_unmap_page(adev->pdev, adev->dummy_page.addr,
 897                         PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 898         __free_page(adev->dummy_page.page);
 899         adev->dummy_page.page = NULL;
 900 }
 901
 902
 903 /* ATOM accessor methods */
 904 /*
 905  * ATOM is an interpreted byte code stored in tables in the vbios.  The
 906  * driver registers callbacks to access registers and the interpreter
 907  * in the driver parses the tables and executes then to program specific
 908  * actions (set display modes, asic init, etc.).  See amdgpu_atombios.c,
 909  * atombios.h, and atom.c
 910  */
 911
 912 /**
 913  * cail_pll_read - read PLL register
 914  *
 915  * @info: atom card_info pointer
 916  * @reg: PLL register offset
 917  *
 918  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 919  * Returns the value of the PLL register.
 920  */
 921 static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
 922 {
 923         return 0;
 924 }
 925
 926 /**
 927  * cail_pll_write - write PLL register
 928  *
 929  * @info: atom card_info pointer
 930  * @reg: PLL register offset
 931  * @val: value to write to the pll register
 932  *
 933  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 934  */
 935 static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
 936 {
 937
 938 }
 939
 940 /**
 941  * cail_mc_read - read MC (Memory Controller) register
 942  *
 943  * @info: atom card_info pointer
 944  * @reg: MC register offset
 945  *
 946  * Provides an MC register accessor for the atom interpreter (r4xx+).
 947  * Returns the value of the MC register.
 948  */
 949 static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
 950 {
 951         return 0;
 952 }
 953
 954 /**
 955  * cail_mc_write - write MC (Memory Controller) register
 956  *
 957  * @info: atom card_info pointer
 958  * @reg: MC register offset
 959  * @val: value to write to the pll register
 960  *
 961  * Provides a MC register accessor for the atom interpreter (r4xx+).
 962  */
 963 static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
 964 {
 965
 966 }
 967
 968 /**
 969  * cail_reg_write - write MMIO register
 970  *
 971  * @info: atom card_info pointer
 972  * @reg: MMIO register offset
 973  * @val: value to write to the pll register
 974  *
 975  * Provides a MMIO register accessor for the atom interpreter (r4xx+).
 976  */
 977 static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
 978 {
 979         struct amdgpu_device *adev = info->dev->dev_private;
 980
 981         WREG32(reg, val);
 982 }
 983
 984 /**
 985  * cail_reg_read - read MMIO register
 986  *
 987  * @info: atom card_info pointer
 988  * @reg: MMIO register offset
 989  *
 990  * Provides an MMIO register accessor for the atom interpreter (r4xx+).
 991  * Returns the value of the MMIO register.
 992  */
 993 static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
 994 {
 995         struct amdgpu_device *adev = info->dev->dev_private;
 996         uint32_t r;
 997
 998         r = RREG32(reg);
 999         return r;
1000 }
1001
1002 /**
1003  * cail_ioreg_write - write IO register
1004  *
1005  * @info: atom card_info pointer
1006  * @reg: IO register offset
1007  * @val: value to write to the pll register
1008  *
1009  * Provides a IO register accessor for the atom interpreter (r4xx+).
1010  */
1011 static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
1012 {
1013         struct amdgpu_device *adev = info->dev->dev_private;
1014
1015         WREG32_IO(reg, val);
1016 }
1017
1018 /**
1019  * cail_ioreg_read - read IO register
1020  *
1021  * @info: atom card_info pointer
1022  * @reg: IO register offset
1023  *
1024  * Provides an IO register accessor for the atom interpreter (r4xx+).
1025  * Returns the value of the IO register.
1026  */
1027 static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
1028 {
1029         struct amdgpu_device *adev = info->dev->dev_private;
1030         uint32_t r;
1031
1032         r = RREG32_IO(reg);
1033         return r;
1034 }
1035
1036 static ssize_t amdgpu_atombios_get_vbios_version(struct device *dev,
1037                                                  struct device_attribute *attr,
1038                                                  char *buf)
1039 {
1040         struct drm_device *ddev = dev_get_drvdata(dev);
1041         struct amdgpu_device *adev = ddev->dev_private;
1042         struct atom_context *ctx = adev->mode_info.atom_context;
1043
1044         return snprintf(buf, PAGE_SIZE, "%s\n", ctx->vbios_version);
1045 }
1046
1047 static DEVICE_ATTR(vbios_version, 0444, amdgpu_atombios_get_vbios_version,
1048                    NULL);
1049
1050 /**
1051  * amdgpu_atombios_fini - free the driver info and callbacks for atombios
1052  *
1053  * @adev: amdgpu_device pointer
1054  *
1055  * Frees the driver info and register access callbacks for the ATOM
1056  * interpreter (r4xx+).
1057  * Called at driver shutdown.
1058  */
1059 static void amdgpu_atombios_fini(struct amdgpu_device *adev)
1060 {
1061         if (adev->mode_info.atom_context) {
1062                 kfree(adev->mode_info.atom_context->scratch);
1063                 kfree(adev->mode_info.atom_context->iio);
1064         }
1065         kfree(adev->mode_info.atom_context);
1066         adev->mode_info.atom_context = NULL;
1067         kfree(adev->mode_info.atom_card_info);
1068         adev->mode_info.atom_card_info = NULL;
1069         device_remove_file(adev->dev, &dev_attr_vbios_version);
1070 }
1071
1072 /**
1073  * amdgpu_atombios_init - init the driver info and callbacks for atombios
1074  *
1075  * @adev: amdgpu_device pointer
1076  *
1077  * Initializes the driver info and register access callbacks for the
1078  * ATOM interpreter (r4xx+).
1079  * Returns 0 on sucess, -ENOMEM on failure.
1080  * Called at driver startup.
1081  */
1082 static int amdgpu_atombios_init(struct amdgpu_device *adev)
1083 {
1084         struct card_info *atom_card_info =
1085             kzalloc(sizeof(struct card_info), GFP_KERNEL);
1086         int ret;
1087
1088         if (!atom_card_info)
1089                 return -ENOMEM;
1090
1091         adev->mode_info.atom_card_info = atom_card_info;
1092         atom_card_info->dev = adev->ddev;
1093         atom_card_info->reg_read = cail_reg_read;
1094         atom_card_info->reg_write = cail_reg_write;
1095         /* needed for iio ops */
1096         if (adev->rio_mem) {
1097                 atom_card_info->ioreg_read = cail_ioreg_read;
1098                 atom_card_info->ioreg_write = cail_ioreg_write;
1099         } else {
1100                 DRM_DEBUG("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
1101                 atom_card_info->ioreg_read = cail_reg_read;
1102                 atom_card_info->ioreg_write = cail_reg_write;
1103         }
1104         atom_card_info->mc_read = cail_mc_read;
1105         atom_card_info->mc_write = cail_mc_write;
1106         atom_card_info->pll_read = cail_pll_read;
1107         atom_card_info->pll_write = cail_pll_write;
1108
1109         adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
1110         if (!adev->mode_info.atom_context) {
1111                 amdgpu_atombios_fini(adev);
1112                 return -ENOMEM;
1113         }
1114
1115         mutex_init(&adev->mode_info.atom_context->mutex);
1116         if (adev->is_atom_fw) {
1117                 amdgpu_atomfirmware_scratch_regs_init(adev);
1118                 amdgpu_atomfirmware_allocate_fb_scratch(adev);
1119         } else {
1120                 amdgpu_atombios_scratch_regs_init(adev);
1121                 amdgpu_atombios_allocate_fb_scratch(adev);
1122         }
1123
1124         ret = device_create_file(adev->dev, &dev_attr_vbios_version);
1125         if (ret) {
1126                 DRM_ERROR("Failed to create device file for VBIOS version\n");
1127                 return ret;
1128         }
1129
1130         return 0;
1131 }
1132
1133 /* if we get transitioned to only one device, take VGA back */
1134 /**
1135  * amdgpu_vga_set_decode - enable/disable vga decode
1136  *
1137  * @cookie: amdgpu_device pointer
1138  * @state: enable/disable vga decode
1139  *
1140  * Enable/disable vga decode (all asics).
1141  * Returns VGA resource flags.
1142  */
1143 static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
1144 {
1145         struct amdgpu_device *adev = cookie;
1146         amdgpu_asic_set_vga_state(adev, state);
1147         if (state)
1148                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1149                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1150         else
1151                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1152 }
1153
1154 static void amdgpu_check_block_size(struct amdgpu_device *adev)
1155 {
1156         /* defines number of bits in page table versus page directory,
1157          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1158          * page table and the remaining bits are in the page directory */
1159         if (amdgpu_vm_block_size == -1)
1160                 return;
1161
1162         if (amdgpu_vm_block_size < 9) {
1163                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1164                          amdgpu_vm_block_size);
1165                 amdgpu_vm_block_size = -1;
1166         }
1167 }
1168
1169 static void amdgpu_check_vm_size(struct amdgpu_device *adev)
1170 {
1171         /* no need to check the default value */
1172         if (amdgpu_vm_size == -1)
1173                 return;
1174
1175         if (amdgpu_vm_size < 1) {
1176                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1177                          amdgpu_vm_size);
1178                 amdgpu_vm_size = -1;
1179         }
1180 }
1181
1182 /**
1183  * amdgpu_check_arguments - validate module params
1184  *
1185  * @adev: amdgpu_device pointer
1186  *
1187  * Validates certain module parameters and updates
1188  * the associated values used by the driver (all asics).
1189  */
1190 static void amdgpu_check_arguments(struct amdgpu_device *adev)
1191 {
1192         if (amdgpu_sched_jobs < 4) {
1193                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1194                          amdgpu_sched_jobs);
1195                 amdgpu_sched_jobs = 4;
1196         } else if (!is_power_of_2(amdgpu_sched_jobs)){
1197                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1198                          amdgpu_sched_jobs);
1199                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1200         }
1201
1202         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1203                 /* gart size must be greater or equal to 32M */
1204                 dev_warn(adev->dev, "gart size (%d) too small\n",
1205                          amdgpu_gart_size);
1206                 amdgpu_gart_size = -1;
1207         }
1208
1209         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1210                 /* gtt size must be greater or equal to 32M */
1211                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1212                                  amdgpu_gtt_size);
1213                 amdgpu_gtt_size = -1;
1214         }
1215
1216         /* valid range is between 4 and 9 inclusive */
1217         if (amdgpu_vm_fragment_size != -1 &&
1218             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1219                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1220                 amdgpu_vm_fragment_size = -1;
1221         }
1222
1223         amdgpu_check_vm_size(adev);
1224
1225         amdgpu_check_block_size(adev);
1226
1227         if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
1228             !is_power_of_2(amdgpu_vram_page_split))) {
1229                 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
1230                          amdgpu_vram_page_split);
1231                 amdgpu_vram_page_split = 1024;
1232         }
1233 }
1234
1235 /**
1236  * amdgpu_switcheroo_set_state - set switcheroo state
1237  *
1238  * @pdev: pci dev pointer
1239  * @state: vga_switcheroo state
1240  *
1241  * Callback for the switcheroo driver.  Suspends or resumes the
1242  * the asics before or after it is powered up using ACPI methods.
1243  */
1244 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1245 {
1246         struct drm_device *dev = pci_get_drvdata(pdev);
1247
1248         if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1249                 return;
1250
1251         if (state == VGA_SWITCHEROO_ON) {
1252                 pr_info("amdgpu: switched on\n");
1253                 /* don't suspend or resume card normally */
1254                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1255
1256                 amdgpu_device_resume(dev, true, true);
1257
1258                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1259                 drm_kms_helper_poll_enable(dev);
1260         } else {
1261                 pr_info("amdgpu: switched off\n");
1262                 drm_kms_helper_poll_disable(dev);
1263                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1264                 amdgpu_device_suspend(dev, true, true);
1265                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1266         }
1267 }
1268
1269 /**
1270  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1271  *
1272  * @pdev: pci dev pointer
1273  *
1274  * Callback for the switcheroo driver.  Check of the switcheroo
1275  * state can be changed.
1276  * Returns true if the state can be changed, false if not.
1277  */
1278 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1279 {
1280         struct drm_device *dev = pci_get_drvdata(pdev);
1281
1282         /*
1283         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1284         * locking inversion with the driver load path. And the access here is
1285         * completely racy anyway. So don't bother with locking for now.
1286         */
1287         return dev->open_count == 0;
1288 }
1289
1290 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1291         .set_gpu_state = amdgpu_switcheroo_set_state,
1292         .reprobe = NULL,
1293         .can_switch = amdgpu_switcheroo_can_switch,
1294 };
1295
1296 int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
1297                                   enum amd_ip_block_type block_type,
1298                                   enum amd_clockgating_state state)
1299 {
1300         int i, r = 0;
1301
1302         for (i = 0; i < adev->num_ip_blocks; i++) {
1303                 if (!adev->ip_blocks[i].status.valid)
1304                         continue;
1305                 if (adev->ip_blocks[i].version->type != block_type)
1306                         continue;
1307                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1308                         continue;
1309                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1310                         (void *)adev, state);
1311                 if (r)
1312                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1313                                   adev->ip_blocks[i].version->funcs->name, r);
1314         }
1315         return r;
1316 }
1317
1318 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
1319                                   enum amd_ip_block_type block_type,
1320                                   enum amd_powergating_state state)
1321 {
1322         int i, r = 0;
1323
1324         for (i = 0; i < adev->num_ip_blocks; i++) {
1325                 if (!adev->ip_blocks[i].status.valid)
1326                         continue;
1327                 if (adev->ip_blocks[i].version->type != block_type)
1328                         continue;
1329                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1330                         continue;
1331                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1332                         (void *)adev, state);
1333                 if (r)
1334                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1335                                   adev->ip_blocks[i].version->funcs->name, r);
1336         }
1337         return r;
1338 }
1339
1340 void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
1341 {
1342         int i;
1343
1344         for (i = 0; i < adev->num_ip_blocks; i++) {
1345                 if (!adev->ip_blocks[i].status.valid)
1346                         continue;
1347                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1348                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1349         }
1350 }
1351
1352 int amdgpu_wait_for_idle(struct amdgpu_device *adev,
1353                          enum amd_ip_block_type block_type)
1354 {
1355         int i, r;
1356
1357         for (i = 0; i < adev->num_ip_blocks; i++) {
1358                 if (!adev->ip_blocks[i].status.valid)
1359                         continue;
1360                 if (adev->ip_blocks[i].version->type == block_type) {
1361                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1362                         if (r)
1363                                 return r;
1364                         break;
1365                 }
1366         }
1367         return 0;
1368
1369 }
1370
1371 bool amdgpu_is_idle(struct amdgpu_device *adev,
1372                     enum amd_ip_block_type block_type)
1373 {
1374         int i;
1375
1376         for (i = 0; i < adev->num_ip_blocks; i++) {
1377                 if (!adev->ip_blocks[i].status.valid)
1378                         continue;
1379                 if (adev->ip_blocks[i].version->type == block_type)
1380                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1381         }
1382         return true;
1383
1384 }
1385
1386 struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
1387                                              enum amd_ip_block_type type)
1388 {
1389         int i;
1390
1391         for (i = 0; i < adev->num_ip_blocks; i++)
1392                 if (adev->ip_blocks[i].version->type == type)
1393                         return &adev->ip_blocks[i];
1394
1395         return NULL;
1396 }
1397
1398 /**
1399  * amdgpu_ip_block_version_cmp
1400  *
1401  * @adev: amdgpu_device pointer
1402  * @type: enum amd_ip_block_type
1403  * @major: major version
1404  * @minor: minor version
1405  *
1406  * return 0 if equal or greater
1407  * return 1 if smaller or the ip_block doesn't exist
1408  */
1409 int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
1410                                 enum amd_ip_block_type type,
1411                                 u32 major, u32 minor)
1412 {
1413         struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
1414
1415         if (ip_block && ((ip_block->version->major > major) ||
1416                         ((ip_block->version->major == major) &&
1417                         (ip_block->version->minor >= minor))))
1418                 return 0;
1419
1420         return 1;
1421 }
1422
1423 /**
1424  * amdgpu_ip_block_add
1425  *
1426  * @adev: amdgpu_device pointer
1427  * @ip_block_version: pointer to the IP to add
1428  *
1429  * Adds the IP block driver information to the collection of IPs
1430  * on the asic.
1431  */
1432 int amdgpu_ip_block_add(struct amdgpu_device *adev,
1433                         const struct amdgpu_ip_block_version *ip_block_version)
1434 {
1435         if (!ip_block_version)
1436                 return -EINVAL;
1437
1438         DRM_DEBUG("add ip block number %d <%s>\n", adev->num_ip_blocks,
1439                   ip_block_version->funcs->name);
1440
1441         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1442
1443         return 0;
1444 }
1445
1446 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1447 {
1448         adev->enable_virtual_display = false;
1449
1450         if (amdgpu_virtual_display) {
1451                 struct drm_device *ddev = adev->ddev;
1452                 const char *pci_address_name = pci_name(ddev->pdev);
1453                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1454
1455                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1456                 pciaddstr_tmp = pciaddstr;
1457                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1458                         pciaddname = strsep(&pciaddname_tmp, ",");
1459                         if (!strcmp("all", pciaddname)
1460                             || !strcmp(pci_address_name, pciaddname)) {
1461                                 long num_crtc;
1462                                 int res = -1;
1463
1464                                 adev->enable_virtual_display = true;
1465
1466                                 if (pciaddname_tmp)
1467                                         res = kstrtol(pciaddname_tmp, 10,
1468                                                       &num_crtc);
1469
1470                                 if (!res) {
1471                                         if (num_crtc < 1)
1472                                                 num_crtc = 1;
1473                                         if (num_crtc > 6)
1474                                                 num_crtc = 6;
1475                                         adev->mode_info.num_crtc = num_crtc;
1476                                 } else {
1477                                         adev->mode_info.num_crtc = 1;
1478                                 }
1479                                 break;
1480                         }
1481                 }
1482
1483                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1484                          amdgpu_virtual_display, pci_address_name,
1485                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1486
1487                 kfree(pciaddstr);
1488         }
1489 }
1490
1491 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1492 {
1493         const char *chip_name;
1494         char fw_name[30];
1495         int err;
1496         const struct gpu_info_firmware_header_v1_0 *hdr;
1497
1498         adev->firmware.gpu_info_fw = NULL;
1499
1500         switch (adev->asic_type) {
1501         case CHIP_TOPAZ:
1502         case CHIP_TONGA:
1503         case CHIP_FIJI:
1504         case CHIP_POLARIS11:
1505         case CHIP_POLARIS10:
1506         case CHIP_POLARIS12:
1507         case CHIP_CARRIZO:
1508         case CHIP_STONEY:
1509 #ifdef CONFIG_DRM_AMDGPU_SI
1510         case CHIP_VERDE:
1511         case CHIP_TAHITI:
1512         case CHIP_PITCAIRN:
1513         case CHIP_OLAND:
1514         case CHIP_HAINAN:
1515 #endif
1516 #ifdef CONFIG_DRM_AMDGPU_CIK
1517         case CHIP_BONAIRE:
1518         case CHIP_HAWAII:
1519         case CHIP_KAVERI:
1520         case CHIP_KABINI:
1521         case CHIP_MULLINS:
1522 #endif
1523         default:
1524                 return 0;
1525         case CHIP_VEGA10:
1526                 chip_name = "vega10";
1527                 break;
1528         case CHIP_RAVEN:
1529                 chip_name = "raven";
1530                 break;
1531         }
1532
1533         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1534         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1535         if (err) {
1536                 dev_err(adev->dev,
1537                         "Failed to load gpu_info firmware \"%s\"\n",
1538                         fw_name);
1539                 goto out;
1540         }
1541         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1542         if (err) {
1543                 dev_err(adev->dev,
1544                         "Failed to validate gpu_info firmware \"%s\"\n",
1545                         fw_name);
1546                 goto out;
1547         }
1548
1549         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1550         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1551
1552         switch (hdr->version_major) {
1553         case 1:
1554         {
1555                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1556                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1557                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1558
1559                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1560                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1561                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1562                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1563                 adev->gfx.config.max_texture_channel_caches =
1564                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
1565                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1566                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1567                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1568                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1569                 adev->gfx.config.double_offchip_lds_buf =
1570                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1571                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1572                 adev->gfx.cu_info.max_waves_per_simd =
1573                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1574                 adev->gfx.cu_info.max_scratch_slots_per_cu =
1575                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1576                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1577                 break;
1578         }
1579         default:
1580                 dev_err(adev->dev,
1581                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1582                 err = -EINVAL;
1583                 goto out;
1584         }
1585 out:
1586         return err;
1587 }
1588
1589 static int amdgpu_early_init(struct amdgpu_device *adev)
1590 {
1591         int i, r;
1592
1593         amdgpu_device_enable_virtual_display(adev);
1594
1595         switch (adev->asic_type) {
1596         case CHIP_TOPAZ:
1597         case CHIP_TONGA:
1598         case CHIP_FIJI:
1599         case CHIP_POLARIS11:
1600         case CHIP_POLARIS10:
1601         case CHIP_POLARIS12:
1602         case CHIP_CARRIZO:
1603         case CHIP_STONEY:
1604                 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1605                         adev->family = AMDGPU_FAMILY_CZ;
1606                 else
1607                         adev->family = AMDGPU_FAMILY_VI;
1608
1609                 r = vi_set_ip_blocks(adev);
1610                 if (r)
1611                         return r;
1612                 break;
1613 #ifdef CONFIG_DRM_AMDGPU_SI
1614         case CHIP_VERDE:
1615         case CHIP_TAHITI:
1616         case CHIP_PITCAIRN:
1617         case CHIP_OLAND:
1618         case CHIP_HAINAN:
1619                 adev->family = AMDGPU_FAMILY_SI;
1620                 r = si_set_ip_blocks(adev);
1621                 if (r)
1622                         return r;
1623                 break;
1624 #endif
1625 #ifdef CONFIG_DRM_AMDGPU_CIK
1626         case CHIP_BONAIRE:
1627         case CHIP_HAWAII:
1628         case CHIP_KAVERI:
1629         case CHIP_KABINI:
1630         case CHIP_MULLINS:
1631                 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1632                         adev->family = AMDGPU_FAMILY_CI;
1633                 else
1634                         adev->family = AMDGPU_FAMILY_KV;
1635
1636                 r = cik_set_ip_blocks(adev);
1637                 if (r)
1638                         return r;
1639                 break;
1640 #endif
1641         case  CHIP_VEGA10:
1642         case  CHIP_RAVEN:
1643                 if (adev->asic_type == CHIP_RAVEN)
1644                         adev->family = AMDGPU_FAMILY_RV;
1645                 else
1646                         adev->family = AMDGPU_FAMILY_AI;
1647
1648                 r = soc15_set_ip_blocks(adev);
1649                 if (r)
1650                         return r;
1651                 break;
1652         default:
1653                 /* FIXME: not supported yet */
1654                 return -EINVAL;
1655         }
1656
1657         r = amdgpu_device_parse_gpu_info_fw(adev);
1658         if (r)
1659                 return r;
1660
1661         amdgpu_amdkfd_device_probe(adev);
1662
1663         if (amdgpu_sriov_vf(adev)) {
1664                 r = amdgpu_virt_request_full_gpu(adev, true);
1665                 if (r)
1666                         return -EAGAIN;
1667         }
1668
1669         for (i = 0; i < adev->num_ip_blocks; i++) {
1670                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1671                         DRM_ERROR("disabled ip block: %d <%s>\n",
1672                                   i, adev->ip_blocks[i].version->funcs->name);
1673                         adev->ip_blocks[i].status.valid = false;
1674                 } else {
1675                         if (adev->ip_blocks[i].version->funcs->early_init) {
1676                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1677                                 if (r == -ENOENT) {
1678                                         adev->ip_blocks[i].status.valid = false;
1679                                 } else if (r) {
1680                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
1681                                                   adev->ip_blocks[i].version->funcs->name, r);
1682                                         return r;
1683                                 } else {
1684                                         adev->ip_blocks[i].status.valid = true;
1685                                 }
1686                         } else {
1687                                 adev->ip_blocks[i].status.valid = true;
1688                         }
1689                 }
1690         }
1691
1692         adev->cg_flags &= amdgpu_cg_mask;
1693         adev->pg_flags &= amdgpu_pg_mask;
1694
1695         return 0;
1696 }
1697
1698 static int amdgpu_init(struct amdgpu_device *adev)
1699 {
1700         int i, r;
1701
1702         for (i = 0; i < adev->num_ip_blocks; i++) {
1703                 if (!adev->ip_blocks[i].status.valid)
1704                         continue;
1705                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1706                 if (r) {
1707                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1708                                   adev->ip_blocks[i].version->funcs->name, r);
1709                         return r;
1710                 }
1711                 adev->ip_blocks[i].status.sw = true;
1712                 /* need to do gmc hw init early so we can allocate gpu mem */
1713                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1714                         r = amdgpu_vram_scratch_init(adev);
1715                         if (r) {
1716                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1717                                 return r;
1718                         }
1719                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1720                         if (r) {
1721                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
1722                                 return r;
1723                         }
1724                         r = amdgpu_wb_init(adev);
1725                         if (r) {
1726                                 DRM_ERROR("amdgpu_wb_init failed %d\n", r);
1727                                 return r;
1728                         }
1729                         adev->ip_blocks[i].status.hw = true;
1730
1731                         /* right after GMC hw init, we create CSA */
1732                         if (amdgpu_sriov_vf(adev)) {
1733                                 r = amdgpu_allocate_static_csa(adev);
1734                                 if (r) {
1735                                         DRM_ERROR("allocate CSA failed %d\n", r);
1736                                         return r;
1737                                 }
1738                         }
1739                 }
1740         }
1741
1742         for (i = 0; i < adev->num_ip_blocks; i++) {
1743                 if (!adev->ip_blocks[i].status.sw)
1744                         continue;
1745                 /* gmc hw init is done early */
1746                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
1747                         continue;
1748                 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1749                 if (r) {
1750                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1751                                   adev->ip_blocks[i].version->funcs->name, r);
1752                         return r;
1753                 }
1754                 adev->ip_blocks[i].status.hw = true;
1755         }
1756
1757         amdgpu_amdkfd_device_init(adev);
1758
1759         if (amdgpu_sriov_vf(adev))
1760                 amdgpu_virt_release_full_gpu(adev, true);
1761
1762         return 0;
1763 }
1764
1765 static void amdgpu_fill_reset_magic(struct amdgpu_device *adev)
1766 {
1767         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1768 }
1769
1770 static bool amdgpu_check_vram_lost(struct amdgpu_device *adev)
1771 {
1772         return !!memcmp(adev->gart.ptr, adev->reset_magic,
1773                         AMDGPU_RESET_MAGIC_NUM);
1774 }
1775
1776 static int amdgpu_late_set_cg_state(struct amdgpu_device *adev)
1777 {
1778         int i = 0, r;
1779
1780         for (i = 0; i < adev->num_ip_blocks; i++) {
1781                 if (!adev->ip_blocks[i].status.valid)
1782                         continue;
1783                 /* skip CG for VCE/UVD, it's handled specially */
1784                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1785                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1786                         /* enable clockgating to save power */
1787                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1788                                                                                      AMD_CG_STATE_GATE);
1789                         if (r) {
1790                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1791                                           adev->ip_blocks[i].version->funcs->name, r);
1792                                 return r;
1793                         }
1794                 }
1795         }
1796         return 0;
1797 }
1798
1799 static int amdgpu_late_init(struct amdgpu_device *adev)
1800 {
1801         int i = 0, r;
1802
1803         for (i = 0; i < adev->num_ip_blocks; i++) {
1804                 if (!adev->ip_blocks[i].status.valid)
1805                         continue;
1806                 if (adev->ip_blocks[i].version->funcs->late_init) {
1807                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1808                         if (r) {
1809                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1810                                           adev->ip_blocks[i].version->funcs->name, r);
1811                                 return r;
1812                         }
1813                         adev->ip_blocks[i].status.late_initialized = true;
1814                 }
1815         }
1816
1817         mod_delayed_work(system_wq, &adev->late_init_work,
1818                         msecs_to_jiffies(AMDGPU_RESUME_MS));
1819
1820         amdgpu_fill_reset_magic(adev);
1821
1822         return 0;
1823 }
1824
1825 static int amdgpu_fini(struct amdgpu_device *adev)
1826 {
1827         int i, r;
1828
1829         amdgpu_amdkfd_device_fini(adev);
1830         /* need to disable SMC first */
1831         for (i = 0; i < adev->num_ip_blocks; i++) {
1832                 if (!adev->ip_blocks[i].status.hw)
1833                         continue;
1834                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
1835                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1836                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1837                                                                                      AMD_CG_STATE_UNGATE);
1838                         if (r) {
1839                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1840                                           adev->ip_blocks[i].version->funcs->name, r);
1841                                 return r;
1842                         }
1843                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1844                         /* XXX handle errors */
1845                         if (r) {
1846                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1847                                           adev->ip_blocks[i].version->funcs->name, r);
1848                         }
1849                         adev->ip_blocks[i].status.hw = false;
1850                         break;
1851                 }
1852         }
1853
1854         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1855                 if (!adev->ip_blocks[i].status.hw)
1856                         continue;
1857                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1858                         amdgpu_free_static_csa(adev);
1859                         amdgpu_wb_fini(adev);
1860                         amdgpu_vram_scratch_fini(adev);
1861                 }
1862
1863                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1864                         adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1865                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1866                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1867                                                                                      AMD_CG_STATE_UNGATE);
1868                         if (r) {
1869                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1870                                           adev->ip_blocks[i].version->funcs->name, r);
1871                                 return r;
1872                         }
1873                 }
1874
1875                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1876                 /* XXX handle errors */
1877                 if (r) {
1878                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1879                                   adev->ip_blocks[i].version->funcs->name, r);
1880                 }
1881
1882                 adev->ip_blocks[i].status.hw = false;
1883         }
1884
1885         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1886                 if (!adev->ip_blocks[i].status.sw)
1887                         continue;
1888                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1889                 /* XXX handle errors */
1890                 if (r) {
1891                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1892                                   adev->ip_blocks[i].version->funcs->name, r);
1893                 }
1894                 adev->ip_blocks[i].status.sw = false;
1895                 adev->ip_blocks[i].status.valid = false;
1896         }
1897
1898         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1899                 if (!adev->ip_blocks[i].status.late_initialized)
1900                         continue;
1901                 if (adev->ip_blocks[i].version->funcs->late_fini)
1902                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1903                 adev->ip_blocks[i].status.late_initialized = false;
1904         }
1905
1906         if (amdgpu_sriov_vf(adev))
1907                 if (amdgpu_virt_release_full_gpu(adev, false))
1908                         DRM_ERROR("failed to release exclusive mode on fini\n");
1909
1910         return 0;
1911 }
1912
1913 static void amdgpu_late_init_func_handler(struct work_struct *work)
1914 {
1915         struct amdgpu_device *adev =
1916                 container_of(work, struct amdgpu_device, late_init_work.work);
1917         amdgpu_late_set_cg_state(adev);
1918 }
1919
1920 int amdgpu_suspend(struct amdgpu_device *adev)
1921 {
1922         int i, r;
1923
1924         if (amdgpu_sriov_vf(adev))
1925                 amdgpu_virt_request_full_gpu(adev, false);
1926
1927         /* ungate SMC block first */
1928         r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1929                                          AMD_CG_STATE_UNGATE);
1930         if (r) {
1931                 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
1932         }
1933
1934         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1935                 if (!adev->ip_blocks[i].status.valid)
1936                         continue;
1937                 /* ungate blocks so that suspend can properly shut them down */
1938                 if (i != AMD_IP_BLOCK_TYPE_SMC) {
1939                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1940                                                                                      AMD_CG_STATE_UNGATE);
1941                         if (r) {
1942                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1943                                           adev->ip_blocks[i].version->funcs->name, r);
1944                         }
1945                 }
1946                 /* XXX handle errors */
1947                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1948                 /* XXX handle errors */
1949                 if (r) {
1950                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
1951                                   adev->ip_blocks[i].version->funcs->name, r);
1952                 }
1953         }
1954
1955         if (amdgpu_sriov_vf(adev))
1956                 amdgpu_virt_release_full_gpu(adev, false);
1957
1958         return 0;
1959 }
1960
1961 static int amdgpu_sriov_reinit_early(struct amdgpu_device *adev)
1962 {
1963         int i, r;
1964
1965         static enum amd_ip_block_type ip_order[] = {
1966                 AMD_IP_BLOCK_TYPE_GMC,
1967                 AMD_IP_BLOCK_TYPE_COMMON,
1968                 AMD_IP_BLOCK_TYPE_IH,
1969         };
1970
1971         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1972                 int j;
1973                 struct amdgpu_ip_block *block;
1974
1975                 for (j = 0; j < adev->num_ip_blocks; j++) {
1976                         block = &adev->ip_blocks[j];
1977
1978                         if (block->version->type != ip_order[i] ||
1979                                 !block->status.valid)
1980                                 continue;
1981
1982                         r = block->version->funcs->hw_init(adev);
1983                         DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1984                 }
1985         }
1986
1987         return 0;
1988 }
1989
1990 static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
1991 {
1992         int i, r;
1993
1994         static enum amd_ip_block_type ip_order[] = {
1995                 AMD_IP_BLOCK_TYPE_SMC,
1996                 AMD_IP_BLOCK_TYPE_PSP,
1997                 AMD_IP_BLOCK_TYPE_DCE,
1998                 AMD_IP_BLOCK_TYPE_GFX,
1999                 AMD_IP_BLOCK_TYPE_SDMA,
2000                 AMD_IP_BLOCK_TYPE_UVD,
2001                 AMD_IP_BLOCK_TYPE_VCE
2002         };
2003
2004         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2005                 int j;
2006                 struct amdgpu_ip_block *block;
2007
2008                 for (j = 0; j < adev->num_ip_blocks; j++) {
2009                         block = &adev->ip_blocks[j];
2010
2011                         if (block->version->type != ip_order[i] ||
2012                                 !block->status.valid)
2013                                 continue;
2014
2015                         r = block->version->funcs->hw_init(adev);
2016                         DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
2017                 }
2018         }
2019
2020         return 0;
2021 }
2022
2023 static int amdgpu_resume_phase1(struct amdgpu_device *adev)
2024 {
2025         int i, r;
2026
2027         for (i = 0; i < adev->num_ip_blocks; i++) {
2028                 if (!adev->ip_blocks[i].status.valid)
2029                         continue;
2030                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2031                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2032                                 adev->ip_blocks[i].version->type ==
2033                                 AMD_IP_BLOCK_TYPE_IH) {
2034                         r = adev->ip_blocks[i].version->funcs->resume(adev);
2035                         if (r) {
2036                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
2037                                           adev->ip_blocks[i].version->funcs->name, r);
2038                                 return r;
2039                         }
2040                 }
2041         }
2042
2043         return 0;
2044 }
2045
2046 static int amdgpu_resume_phase2(struct amdgpu_device *adev)
2047 {
2048         int i, r;
2049
2050         for (i = 0; i < adev->num_ip_blocks; i++) {
2051                 if (!adev->ip_blocks[i].status.valid)
2052                         continue;
2053                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2054                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2055                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH )
2056                         continue;
2057                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2058                 if (r) {
2059                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2060                                   adev->ip_blocks[i].version->funcs->name, r);
2061                         return r;
2062                 }
2063         }
2064
2065         return 0;
2066 }
2067
2068 static int amdgpu_resume(struct amdgpu_device *adev)
2069 {
2070         int r;
2071
2072         r = amdgpu_resume_phase1(adev);
2073         if (r)
2074                 return r;
2075         r = amdgpu_resume_phase2(adev);
2076
2077         return r;
2078 }
2079
2080 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2081 {
2082         if (amdgpu_sriov_vf(adev)) {
2083                 if (adev->is_atom_fw) {
2084                         if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2085                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2086                 } else {
2087                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2088                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2089                 }
2090
2091                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
2092                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2093         }
2094 }
2095
2096 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
2097 {
2098         switch (asic_type) {
2099 #if defined(CONFIG_DRM_AMD_DC)
2100         case CHIP_BONAIRE:
2101         case CHIP_HAWAII:
2102         case CHIP_KAVERI:
2103         case CHIP_CARRIZO:
2104         case CHIP_STONEY:
2105         case CHIP_POLARIS11:
2106         case CHIP_POLARIS10:
2107         case CHIP_POLARIS12:
2108         case CHIP_TONGA:
2109         case CHIP_FIJI:
2110 #if defined(CONFIG_DRM_AMD_DC_PRE_VEGA)
2111                 return amdgpu_dc != 0;
2112 #endif
2113         case CHIP_KABINI:
2114         case CHIP_MULLINS:
2115                 return amdgpu_dc > 0;
2116         case CHIP_VEGA10:
2117 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
2118         case CHIP_RAVEN:
2119 #endif
2120                 return amdgpu_dc != 0;
2121 #endif
2122         default:
2123                 return false;
2124         }
2125 }
2126
2127 /**
2128  * amdgpu_device_has_dc_support - check if dc is supported
2129  *
2130  * @adev: amdgpu_device_pointer
2131  *
2132  * Returns true for supported, false for not supported
2133  */
2134 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
2135 {
2136         if (amdgpu_sriov_vf(adev))
2137                 return false;
2138
2139         return amdgpu_device_asic_has_dc_support(adev->asic_type);
2140 }
2141
2142 /**
2143  * amdgpu_device_init - initialize the driver
2144  *
2145  * @adev: amdgpu_device pointer
2146  * @pdev: drm dev pointer
2147  * @pdev: pci dev pointer
2148  * @flags: driver flags
2149  *
2150  * Initializes the driver info and hw (all asics).
2151  * Returns 0 for success or an error on failure.
2152  * Called at driver startup.
2153  */
2154 int amdgpu_device_init(struct amdgpu_device *adev,
2155                        struct drm_device *ddev,
2156                        struct pci_dev *pdev,
2157                        uint32_t flags)
2158 {
2159         int r, i;
2160         bool runtime = false;
2161         u32 max_MBps;
2162
2163         adev->shutdown = false;
2164         adev->dev = &pdev->dev;
2165         adev->ddev = ddev;
2166         adev->pdev = pdev;
2167         adev->flags = flags;
2168         adev->asic_type = flags & AMD_ASIC_MASK;
2169         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2170         adev->mc.gart_size = 512 * 1024 * 1024;
2171         adev->accel_working = false;
2172         adev->num_rings = 0;
2173         adev->mman.buffer_funcs = NULL;
2174         adev->mman.buffer_funcs_ring = NULL;
2175         adev->vm_manager.vm_pte_funcs = NULL;
2176         adev->vm_manager.vm_pte_num_rings = 0;
2177         adev->gart.gart_funcs = NULL;
2178         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2179         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2180
2181         adev->smc_rreg = &amdgpu_invalid_rreg;
2182         adev->smc_wreg = &amdgpu_invalid_wreg;
2183         adev->pcie_rreg = &amdgpu_invalid_rreg;
2184         adev->pcie_wreg = &amdgpu_invalid_wreg;
2185         adev->pciep_rreg = &amdgpu_invalid_rreg;
2186         adev->pciep_wreg = &amdgpu_invalid_wreg;
2187         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2188         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2189         adev->didt_rreg = &amdgpu_invalid_rreg;
2190         adev->didt_wreg = &amdgpu_invalid_wreg;
2191         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2192         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
2193         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2194         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2195
2196         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2197                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2198                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
2199
2200         /* mutex initialization are all done here so we
2201          * can recall function without having locking issues */
2202         atomic_set(&adev->irq.ih.lock, 0);
2203         mutex_init(&adev->firmware.mutex);
2204         mutex_init(&adev->pm.mutex);
2205         mutex_init(&adev->gfx.gpu_clock_mutex);
2206         mutex_init(&adev->srbm_mutex);
2207         mutex_init(&adev->gfx.pipe_reserve_mutex);
2208         mutex_init(&adev->grbm_idx_mutex);
2209         mutex_init(&adev->mn_lock);
2210         mutex_init(&adev->virt.vf_errors.lock);
2211         hash_init(adev->mn_hash);
2212         mutex_init(&adev->lock_reset);
2213
2214         amdgpu_check_arguments(adev);
2215
2216         spin_lock_init(&adev->mmio_idx_lock);
2217         spin_lock_init(&adev->smc_idx_lock);
2218         spin_lock_init(&adev->pcie_idx_lock);
2219         spin_lock_init(&adev->uvd_ctx_idx_lock);
2220         spin_lock_init(&adev->didt_idx_lock);
2221         spin_lock_init(&adev->gc_cac_idx_lock);
2222         spin_lock_init(&adev->se_cac_idx_lock);
2223         spin_lock_init(&adev->audio_endpt_idx_lock);
2224         spin_lock_init(&adev->mm_stats.lock);
2225
2226         INIT_LIST_HEAD(&adev->shadow_list);
2227         mutex_init(&adev->shadow_list_lock);
2228
2229         INIT_LIST_HEAD(&adev->ring_lru_list);
2230         spin_lock_init(&adev->ring_lru_list_lock);
2231
2232         INIT_DELAYED_WORK(&adev->late_init_work, amdgpu_late_init_func_handler);
2233
2234         /* Registers mapping */
2235         /* TODO: block userspace mapping of io register */
2236         if (adev->asic_type >= CHIP_BONAIRE) {
2237                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2238                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2239         } else {
2240                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2241                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2242         }
2243
2244         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2245         if (adev->rmmio == NULL) {
2246                 return -ENOMEM;
2247         }
2248         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2249         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2250
2251         /* doorbell bar mapping */
2252         amdgpu_doorbell_init(adev);
2253
2254         /* io port mapping */
2255         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2256                 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2257                         adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2258                         adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2259                         break;
2260                 }
2261         }
2262         if (adev->rio_mem == NULL)
2263                 DRM_INFO("PCI I/O BAR is not found.\n");
2264
2265         /* early init functions */
2266         r = amdgpu_early_init(adev);
2267         if (r)
2268                 return r;
2269
2270         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2271         /* this will fail for cards that aren't VGA class devices, just
2272          * ignore it */
2273         vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
2274
2275         if (amdgpu_runtime_pm == 1)
2276                 runtime = true;
2277         if (amdgpu_device_is_px(ddev))
2278                 runtime = true;
2279         if (!pci_is_thunderbolt_attached(adev->pdev))
2280                 vga_switcheroo_register_client(adev->pdev,
2281                                                &amdgpu_switcheroo_ops, runtime);
2282         if (runtime)
2283                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2284
2285         /* Read BIOS */
2286         if (!amdgpu_get_bios(adev)) {
2287                 r = -EINVAL;
2288                 goto failed;
2289         }
2290
2291         r = amdgpu_atombios_init(adev);
2292         if (r) {
2293                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2294                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2295                 goto failed;
2296         }
2297
2298         /* detect if we are with an SRIOV vbios */
2299         amdgpu_device_detect_sriov_bios(adev);
2300
2301         /* Post card if necessary */
2302         if (amdgpu_need_post(adev)) {
2303                 if (!adev->bios) {
2304                         dev_err(adev->dev, "no vBIOS found\n");
2305                         r = -EINVAL;
2306                         goto failed;
2307                 }
2308                 DRM_INFO("GPU posting now...\n");
2309                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2310                 if (r) {
2311                         dev_err(adev->dev, "gpu post error!\n");
2312                         goto failed;
2313                 }
2314         }
2315
2316         if (adev->is_atom_fw) {
2317                 /* Initialize clocks */
2318                 r = amdgpu_atomfirmware_get_clock_info(adev);
2319                 if (r) {
2320                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
2321                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2322                         goto failed;
2323                 }
2324         } else {
2325                 /* Initialize clocks */
2326                 r = amdgpu_atombios_get_clock_info(adev);
2327                 if (r) {
2328                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2329                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2330                         goto failed;
2331                 }
2332                 /* init i2c buses */
2333                 if (!amdgpu_device_has_dc_support(adev))
2334                         amdgpu_atombios_i2c_init(adev);
2335         }
2336
2337         /* Fence driver */
2338         r = amdgpu_fence_driver_init(adev);
2339         if (r) {
2340                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2341                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2342                 goto failed;
2343         }
2344
2345         /* init the mode config */
2346         drm_mode_config_init(adev->ddev);
2347
2348         r = amdgpu_init(adev);
2349         if (r) {
2350                 /* failed in exclusive mode due to timeout */
2351                 if (amdgpu_sriov_vf(adev) &&
2352                     !amdgpu_sriov_runtime(adev) &&
2353                     amdgpu_virt_mmio_blocked(adev) &&
2354                     !amdgpu_virt_wait_reset(adev)) {
2355                         dev_err(adev->dev, "VF exclusive mode timeout\n");
2356                         /* Don't send request since VF is inactive. */
2357                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
2358                         adev->virt.ops = NULL;
2359                         r = -EAGAIN;
2360                         goto failed;
2361                 }
2362                 dev_err(adev->dev, "amdgpu_init failed\n");
2363                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2364                 amdgpu_fini(adev);
2365                 goto failed;
2366         }
2367
2368         adev->accel_working = true;
2369
2370         amdgpu_vm_check_compute_bug(adev);
2371
2372         /* Initialize the buffer migration limit. */
2373         if (amdgpu_moverate >= 0)
2374                 max_MBps = amdgpu_moverate;
2375         else
2376                 max_MBps = 8; /* Allow 8 MB/s. */
2377         /* Get a log2 for easy divisions. */
2378         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2379
2380         r = amdgpu_ib_pool_init(adev);
2381         if (r) {
2382                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2383                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2384                 goto failed;
2385         }
2386
2387         r = amdgpu_ib_ring_tests(adev);
2388         if (r)
2389                 DRM_ERROR("ib ring test failed (%d).\n", r);
2390
2391         if (amdgpu_sriov_vf(adev))
2392                 amdgpu_virt_init_data_exchange(adev);
2393
2394         amdgpu_fbdev_init(adev);
2395
2396         r = amdgpu_pm_sysfs_init(adev);
2397         if (r)
2398                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2399
2400         r = amdgpu_gem_debugfs_init(adev);
2401         if (r)
2402                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2403
2404         r = amdgpu_debugfs_regs_init(adev);
2405         if (r)
2406                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2407
2408         r = amdgpu_debugfs_test_ib_ring_init(adev);
2409         if (r)
2410                 DRM_ERROR("registering register test ib ring debugfs failed (%d).\n", r);
2411
2412         r = amdgpu_debugfs_firmware_init(adev);
2413         if (r)
2414                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2415
2416         r = amdgpu_debugfs_vbios_dump_init(adev);
2417         if (r)
2418                 DRM_ERROR("Creating vbios dump debugfs failed (%d).\n", r);
2419
2420         if ((amdgpu_testing & 1)) {
2421                 if (adev->accel_working)
2422                         amdgpu_test_moves(adev);
2423                 else
2424                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2425         }
2426         if (amdgpu_benchmarking) {
2427                 if (adev->accel_working)
2428                         amdgpu_benchmark(adev, amdgpu_benchmarking);
2429                 else
2430                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2431         }
2432
2433         /* enable clockgating, etc. after ib tests, etc. since some blocks require
2434          * explicit gating rather than handling it automatically.
2435          */
2436         r = amdgpu_late_init(adev);
2437         if (r) {
2438                 dev_err(adev->dev, "amdgpu_late_init failed\n");
2439                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2440                 goto failed;
2441         }
2442
2443         return 0;
2444
2445 failed:
2446         amdgpu_vf_error_trans_all(adev);
2447         if (runtime)
2448                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2449
2450         return r;
2451 }
2452
2453 /**
2454  * amdgpu_device_fini - tear down the driver
2455  *
2456  * @adev: amdgpu_device pointer
2457  *
2458  * Tear down the driver info (all asics).
2459  * Called at driver shutdown.
2460  */
2461 void amdgpu_device_fini(struct amdgpu_device *adev)
2462 {
2463         int r;
2464
2465         DRM_INFO("amdgpu: finishing device.\n");
2466         adev->shutdown = true;
2467         if (adev->mode_info.mode_config_initialized)
2468                 drm_crtc_force_disable_all(adev->ddev);
2469         /* evict vram memory */
2470         amdgpu_bo_evict_vram(adev);
2471         amdgpu_ib_pool_fini(adev);
2472         amdgpu_fence_driver_fini(adev);
2473         amdgpu_fbdev_fini(adev);
2474         r = amdgpu_fini(adev);
2475         if (adev->firmware.gpu_info_fw) {
2476                 release_firmware(adev->firmware.gpu_info_fw);
2477                 adev->firmware.gpu_info_fw = NULL;
2478         }
2479         adev->accel_working = false;
2480         cancel_delayed_work_sync(&adev->late_init_work);
2481         /* free i2c buses */
2482         if (!amdgpu_device_has_dc_support(adev))
2483                 amdgpu_i2c_fini(adev);
2484         amdgpu_atombios_fini(adev);
2485         kfree(adev->bios);
2486         adev->bios = NULL;
2487         if (!pci_is_thunderbolt_attached(adev->pdev))
2488                 vga_switcheroo_unregister_client(adev->pdev);
2489         if (adev->flags & AMD_IS_PX)
2490                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2491         vga_client_register(adev->pdev, NULL, NULL, NULL);
2492         if (adev->rio_mem)
2493                 pci_iounmap(adev->pdev, adev->rio_mem);
2494         adev->rio_mem = NULL;
2495         iounmap(adev->rmmio);
2496         adev->rmmio = NULL;
2497         amdgpu_doorbell_fini(adev);
2498         amdgpu_pm_sysfs_fini(adev);
2499         amdgpu_debugfs_regs_cleanup(adev);
2500 }
2501
2502
2503 /*
2504  * Suspend & resume.
2505  */
2506 /**
2507  * amdgpu_device_suspend - initiate device suspend
2508  *
2509  * @pdev: drm dev pointer
2510  * @state: suspend state
2511  *
2512  * Puts the hw in the suspend state (all asics).
2513  * Returns 0 for success or an error on failure.
2514  * Called at driver suspend.
2515  */
2516 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2517 {
2518         struct amdgpu_device *adev;
2519         struct drm_crtc *crtc;
2520         struct drm_connector *connector;
2521         int r;
2522
2523         if (dev == NULL || dev->dev_private == NULL) {
2524                 return -ENODEV;
2525         }
2526
2527         adev = dev->dev_private;
2528
2529         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2530                 return 0;
2531
2532         drm_kms_helper_poll_disable(dev);
2533
2534         if (!amdgpu_device_has_dc_support(adev)) {
2535                 /* turn off display hw */
2536                 drm_modeset_lock_all(dev);
2537                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2538                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2539                 }
2540                 drm_modeset_unlock_all(dev);
2541         }
2542
2543         amdgpu_amdkfd_suspend(adev);
2544
2545         /* unpin the front buffers and cursors */
2546         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2547                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2548                 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
2549                 struct amdgpu_bo *robj;
2550
2551                 if (amdgpu_crtc->cursor_bo) {
2552                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2553                         r = amdgpu_bo_reserve(aobj, true);
2554                         if (r == 0) {
2555                                 amdgpu_bo_unpin(aobj);
2556                                 amdgpu_bo_unreserve(aobj);
2557                         }
2558                 }
2559
2560                 if (rfb == NULL || rfb->obj == NULL) {
2561                         continue;
2562                 }
2563                 robj = gem_to_amdgpu_bo(rfb->obj);
2564                 /* don't unpin kernel fb objects */
2565                 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2566                         r = amdgpu_bo_reserve(robj, true);
2567                         if (r == 0) {
2568                                 amdgpu_bo_unpin(robj);
2569                                 amdgpu_bo_unreserve(robj);
2570                         }
2571                 }
2572         }
2573         /* evict vram memory */
2574         amdgpu_bo_evict_vram(adev);
2575
2576         amdgpu_fence_driver_suspend(adev);
2577
2578         r = amdgpu_suspend(adev);
2579
2580         /* evict remaining vram memory
2581          * This second call to evict vram is to evict the gart page table
2582          * using the CPU.
2583          */
2584         amdgpu_bo_evict_vram(adev);
2585
2586         amdgpu_atombios_scratch_regs_save(adev);
2587         pci_save_state(dev->pdev);
2588         if (suspend) {
2589                 /* Shut down the device */
2590                 pci_disable_device(dev->pdev);
2591                 pci_set_power_state(dev->pdev, PCI_D3hot);
2592         } else {
2593                 r = amdgpu_asic_reset(adev);
2594                 if (r)
2595                         DRM_ERROR("amdgpu asic reset failed\n");
2596         }
2597
2598         if (fbcon) {
2599                 console_lock();
2600                 amdgpu_fbdev_set_suspend(adev, 1);
2601                 console_unlock();
2602         }
2603         return 0;
2604 }
2605
2606 /**
2607  * amdgpu_device_resume - initiate device resume
2608  *
2609  * @pdev: drm dev pointer
2610  *
2611  * Bring the hw back to operating state (all asics).
2612  * Returns 0 for success or an error on failure.
2613  * Called at driver resume.
2614  */
2615 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2616 {
2617         struct drm_connector *connector;
2618         struct amdgpu_device *adev = dev->dev_private;
2619         struct drm_crtc *crtc;
2620         int r = 0;
2621
2622         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2623                 return 0;
2624
2625         if (fbcon)
2626                 console_lock();
2627
2628         if (resume) {
2629                 pci_set_power_state(dev->pdev, PCI_D0);
2630                 pci_restore_state(dev->pdev);
2631                 r = pci_enable_device(dev->pdev);
2632                 if (r)
2633                         goto unlock;
2634         }
2635         amdgpu_atombios_scratch_regs_restore(adev);
2636
2637         /* post card */
2638         if (amdgpu_need_post(adev)) {
2639                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2640                 if (r)
2641                         DRM_ERROR("amdgpu asic init failed\n");
2642         }
2643
2644         r = amdgpu_resume(adev);
2645         if (r) {
2646                 DRM_ERROR("amdgpu_resume failed (%d).\n", r);
2647                 goto unlock;
2648         }
2649         amdgpu_fence_driver_resume(adev);
2650
2651         if (resume) {
2652                 r = amdgpu_ib_ring_tests(adev);
2653                 if (r)
2654                         DRM_ERROR("ib ring test failed (%d).\n", r);
2655         }
2656
2657         r = amdgpu_late_init(adev);
2658         if (r)
2659                 goto unlock;
2660
2661         /* pin cursors */
2662         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2663                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2664
2665                 if (amdgpu_crtc->cursor_bo) {
2666                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2667                         r = amdgpu_bo_reserve(aobj, true);
2668                         if (r == 0) {
2669                                 r = amdgpu_bo_pin(aobj,
2670                                                   AMDGPU_GEM_DOMAIN_VRAM,
2671                                                   &amdgpu_crtc->cursor_addr);
2672                                 if (r != 0)
2673                                         DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2674                                 amdgpu_bo_unreserve(aobj);
2675                         }
2676                 }
2677         }
2678         r = amdgpu_amdkfd_resume(adev);
2679         if (r)
2680                 return r;
2681
2682         /* blat the mode back in */
2683         if (fbcon) {
2684                 if (!amdgpu_device_has_dc_support(adev)) {
2685                         /* pre DCE11 */
2686                         drm_helper_resume_force_mode(dev);
2687
2688                         /* turn on display hw */
2689                         drm_modeset_lock_all(dev);
2690                         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2691                                 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2692                         }
2693                         drm_modeset_unlock_all(dev);
2694                 } else {
2695                         /*
2696                          * There is no equivalent atomic helper to turn on
2697                          * display, so we defined our own function for this,
2698                          * once suspend resume is supported by the atomic
2699                          * framework this will be reworked
2700                          */
2701                         amdgpu_dm_display_resume(adev);
2702                 }
2703         }
2704
2705         drm_kms_helper_poll_enable(dev);
2706
2707         /*
2708          * Most of the connector probing functions try to acquire runtime pm
2709          * refs to ensure that the GPU is powered on when connector polling is
2710          * performed. Since we're calling this from a runtime PM callback,
2711          * trying to acquire rpm refs will cause us to deadlock.
2712          *
2713          * Since we're guaranteed to be holding the rpm lock, it's safe to
2714          * temporarily disable the rpm helpers so this doesn't deadlock us.
2715          */
2716 #ifdef CONFIG_PM
2717         dev->dev->power.disable_depth++;
2718 #endif
2719         if (!amdgpu_device_has_dc_support(adev))
2720                 drm_helper_hpd_irq_event(dev);
2721         else
2722                 drm_kms_helper_hotplug_event(dev);
2723 #ifdef CONFIG_PM
2724         dev->dev->power.disable_depth--;
2725 #endif
2726
2727         if (fbcon)
2728                 amdgpu_fbdev_set_suspend(adev, 0);
2729
2730 unlock:
2731         if (fbcon)
2732                 console_unlock();
2733
2734         return r;
2735 }
2736
2737 static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
2738 {
2739         int i;
2740         bool asic_hang = false;
2741
2742         if (amdgpu_sriov_vf(adev))
2743                 return true;
2744
2745         for (i = 0; i < adev->num_ip_blocks; i++) {
2746                 if (!adev->ip_blocks[i].status.valid)
2747                         continue;
2748                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2749                         adev->ip_blocks[i].status.hang =
2750                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2751                 if (adev->ip_blocks[i].status.hang) {
2752                         DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2753                         asic_hang = true;
2754                 }
2755         }
2756         return asic_hang;
2757 }
2758
2759 static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
2760 {
2761         int i, r = 0;
2762
2763         for (i = 0; i < adev->num_ip_blocks; i++) {
2764                 if (!adev->ip_blocks[i].status.valid)
2765                         continue;
2766                 if (adev->ip_blocks[i].status.hang &&
2767                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2768                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2769                         if (r)
2770                                 return r;
2771                 }
2772         }
2773
2774         return 0;
2775 }
2776
2777 static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
2778 {
2779         int i;
2780
2781         for (i = 0; i < adev->num_ip_blocks; i++) {
2782                 if (!adev->ip_blocks[i].status.valid)
2783                         continue;
2784                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2785                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2786                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2787                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
2788                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2789                         if (adev->ip_blocks[i].status.hang) {
2790                                 DRM_INFO("Some block need full reset!\n");
2791                                 return true;
2792                         }
2793                 }
2794         }
2795         return false;
2796 }
2797
2798 static int amdgpu_soft_reset(struct amdgpu_device *adev)
2799 {
2800         int i, r = 0;
2801
2802         for (i = 0; i < adev->num_ip_blocks; i++) {
2803                 if (!adev->ip_blocks[i].status.valid)
2804                         continue;
2805                 if (adev->ip_blocks[i].status.hang &&
2806                     adev->ip_blocks[i].version->funcs->soft_reset) {
2807                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2808                         if (r)
2809                                 return r;
2810                 }
2811         }
2812
2813         return 0;
2814 }
2815
2816 static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
2817 {
2818         int i, r = 0;
2819
2820         for (i = 0; i < adev->num_ip_blocks; i++) {
2821                 if (!adev->ip_blocks[i].status.valid)
2822                         continue;
2823                 if (adev->ip_blocks[i].status.hang &&
2824                     adev->ip_blocks[i].version->funcs->post_soft_reset)
2825                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2826                 if (r)
2827                         return r;
2828         }
2829
2830         return 0;
2831 }
2832
2833 bool amdgpu_need_backup(struct amdgpu_device *adev)
2834 {
2835         if (adev->flags & AMD_IS_APU)
2836                 return false;
2837
2838         return amdgpu_lockup_timeout > 0 ? true : false;
2839 }
2840
2841 static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
2842                                            struct amdgpu_ring *ring,
2843                                            struct amdgpu_bo *bo,
2844                                            struct dma_fence **fence)
2845 {
2846         uint32_t domain;
2847         int r;
2848
2849         if (!bo->shadow)
2850                 return 0;
2851
2852         r = amdgpu_bo_reserve(bo, true);
2853         if (r)
2854                 return r;
2855         domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2856         /* if bo has been evicted, then no need to recover */
2857         if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2858                 r = amdgpu_bo_validate(bo->shadow);
2859                 if (r) {
2860                         DRM_ERROR("bo validate failed!\n");
2861                         goto err;
2862                 }
2863
2864                 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2865                                                  NULL, fence, true);
2866                 if (r) {
2867                         DRM_ERROR("recover page table failed!\n");
2868                         goto err;
2869                 }
2870         }
2871 err:
2872         amdgpu_bo_unreserve(bo);
2873         return r;
2874 }
2875
2876 /*
2877  * amdgpu_reset - reset ASIC/GPU for bare-metal or passthrough
2878  *
2879  * @adev: amdgpu device pointer
2880  * @reset_flags: output param tells caller the reset result
2881  *
2882  * attempt to do soft-reset or full-reset and reinitialize Asic
2883  * return 0 means successed otherwise failed
2884 */
2885 static int amdgpu_reset(struct amdgpu_device *adev, uint64_t* reset_flags)
2886 {
2887         bool need_full_reset, vram_lost = 0;
2888         int r;
2889
2890         need_full_reset = amdgpu_need_full_reset(adev);
2891
2892         if (!need_full_reset) {
2893                 amdgpu_pre_soft_reset(adev);
2894                 r = amdgpu_soft_reset(adev);
2895                 amdgpu_post_soft_reset(adev);
2896                 if (r || amdgpu_check_soft_reset(adev)) {
2897                         DRM_INFO("soft reset failed, will fallback to full reset!\n");
2898                         need_full_reset = true;
2899                 }
2900
2901         }
2902
2903         if (need_full_reset) {
2904                 r = amdgpu_suspend(adev);
2905
2906 retry:
2907                 amdgpu_atombios_scratch_regs_save(adev);
2908                 r = amdgpu_asic_reset(adev);
2909                 amdgpu_atombios_scratch_regs_restore(adev);
2910                 /* post card */
2911                 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2912
2913                 if (!r) {
2914                         dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
2915                         r = amdgpu_resume_phase1(adev);
2916                         if (r)
2917                                 goto out;
2918
2919                         vram_lost = amdgpu_check_vram_lost(adev);
2920                         if (vram_lost) {
2921                                 DRM_ERROR("VRAM is lost!\n");
2922                                 atomic_inc(&adev->vram_lost_counter);
2923                         }
2924
2925                         r = amdgpu_gtt_mgr_recover(
2926                                 &adev->mman.bdev.man[TTM_PL_TT]);
2927                         if (r)
2928                                 goto out;
2929
2930                         r = amdgpu_resume_phase2(adev);
2931                         if (r)
2932                                 goto out;
2933
2934                         if (vram_lost)
2935                                 amdgpu_fill_reset_magic(adev);
2936                 }
2937         }
2938
2939 out:
2940         if (!r) {
2941                 amdgpu_irq_gpu_reset_resume_helper(adev);
2942                 r = amdgpu_ib_ring_tests(adev);
2943                 if (r) {
2944                         dev_err(adev->dev, "ib ring test failed (%d).\n", r);
2945                         r = amdgpu_suspend(adev);
2946                         need_full_reset = true;
2947                         goto retry;
2948                 }
2949         }
2950
2951         if (reset_flags) {
2952                 if (vram_lost)
2953                         (*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
2954
2955                 if (need_full_reset)
2956                         (*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
2957         }
2958
2959         return r;
2960 }
2961
2962 /*
2963  * amdgpu_reset_sriov - reset ASIC for SR-IOV vf
2964  *
2965  * @adev: amdgpu device pointer
2966  * @reset_flags: output param tells caller the reset result
2967  *
2968  * do VF FLR and reinitialize Asic
2969  * return 0 means successed otherwise failed
2970 */
2971 static int amdgpu_reset_sriov(struct amdgpu_device *adev, uint64_t *reset_flags, bool from_hypervisor)
2972 {
2973         int r;
2974
2975         if (from_hypervisor)
2976                 r = amdgpu_virt_request_full_gpu(adev, true);
2977         else
2978                 r = amdgpu_virt_reset_gpu(adev);
2979         if (r)
2980                 return r;
2981
2982         /* Resume IP prior to SMC */
2983         r = amdgpu_sriov_reinit_early(adev);
2984         if (r)
2985                 goto error;
2986
2987         /* we need recover gart prior to run SMC/CP/SDMA resume */
2988         amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
2989
2990         /* now we are okay to resume SMC/CP/SDMA */
2991         r = amdgpu_sriov_reinit_late(adev);
2992         if (r)
2993                 goto error;
2994
2995         amdgpu_irq_gpu_reset_resume_helper(adev);
2996         r = amdgpu_ib_ring_tests(adev);
2997         if (r)
2998                 dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
2999
3000 error:
3001         /* release full control of GPU after ib test */
3002         amdgpu_virt_release_full_gpu(adev, true);
3003
3004         if (reset_flags) {
3005                 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
3006                         (*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
3007                         atomic_inc(&adev->vram_lost_counter);
3008                 }
3009
3010                 /* VF FLR or hotlink reset is always full-reset */
3011                 (*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
3012         }
3013
3014         return r;
3015 }
3016
3017 /**
3018  * amdgpu_gpu_recover - reset the asic and recover scheduler
3019  *
3020  * @adev: amdgpu device pointer
3021  * @job: which job trigger hang
3022  *
3023  * Attempt to reset the GPU if it has hung (all asics).
3024  * Returns 0 for success or an error on failure.
3025  */
3026 int amdgpu_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job *job)
3027 {
3028         struct drm_atomic_state *state = NULL;
3029         uint64_t reset_flags = 0;
3030         int i, r, resched;
3031
3032         if (!amdgpu_check_soft_reset(adev)) {
3033                 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
3034                 return 0;
3035         }
3036
3037         dev_info(adev->dev, "GPU reset begin!\n");
3038
3039         mutex_lock(&adev->lock_reset);
3040         atomic_inc(&adev->gpu_reset_counter);
3041         adev->in_gpu_reset = 1;
3042
3043         /* block TTM */
3044         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3045         /* store modesetting */
3046         if (amdgpu_device_has_dc_support(adev))
3047                 state = drm_atomic_helper_suspend(adev->ddev);
3048
3049         /* block scheduler */
3050         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3051                 struct amdgpu_ring *ring = adev->rings[i];
3052
3053                 if (!ring || !ring->sched.thread)
3054                         continue;
3055
3056                 /* only focus on the ring hit timeout if &job not NULL */
3057                 if (job && job->ring->idx != i)
3058                         continue;
3059
3060                 kthread_park(ring->sched.thread);
3061                 amd_sched_hw_job_reset(&ring->sched, &job->base);
3062
3063                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
3064                 amdgpu_fence_driver_force_completion(ring);
3065         }
3066
3067         if (amdgpu_sriov_vf(adev))
3068                 r = amdgpu_reset_sriov(adev, &reset_flags, job ? false : true);
3069         else
3070                 r = amdgpu_reset(adev, &reset_flags);
3071
3072         if (!r) {
3073                 if (((reset_flags & AMDGPU_RESET_INFO_FULLRESET) && !(adev->flags & AMD_IS_APU)) ||
3074                         (reset_flags & AMDGPU_RESET_INFO_VRAM_LOST)) {
3075                         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
3076                         struct amdgpu_bo *bo, *tmp;
3077                         struct dma_fence *fence = NULL, *next = NULL;
3078
3079                         DRM_INFO("recover vram bo from shadow\n");
3080                         mutex_lock(&adev->shadow_list_lock);
3081                         list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
3082                                 next = NULL;
3083                                 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
3084                                 if (fence) {
3085                                         r = dma_fence_wait(fence, false);
3086                                         if (r) {
3087                                                 WARN(r, "recovery from shadow isn't completed\n");
3088                                                 break;
3089                                         }
3090                                 }
3091
3092                                 dma_fence_put(fence);
3093                                 fence = next;
3094                         }
3095                         mutex_unlock(&adev->shadow_list_lock);
3096                         if (fence) {
3097                                 r = dma_fence_wait(fence, false);
3098                                 if (r)
3099                                         WARN(r, "recovery from shadow isn't completed\n");
3100                         }
3101                         dma_fence_put(fence);
3102                 }
3103
3104                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3105                         struct amdgpu_ring *ring = adev->rings[i];
3106
3107                         if (!ring || !ring->sched.thread)
3108                                 continue;
3109
3110                         /* only focus on the ring hit timeout if &job not NULL */
3111                         if (job && job->ring->idx != i)
3112                                 continue;
3113
3114                         amd_sched_job_recovery(&ring->sched);
3115                         kthread_unpark(ring->sched.thread);
3116                 }
3117         } else {
3118                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3119                         struct amdgpu_ring *ring = adev->rings[i];
3120
3121                         if (!ring || !ring->sched.thread)
3122                                 continue;
3123
3124                         /* only focus on the ring hit timeout if &job not NULL */
3125                         if (job && job->ring->idx != i)
3126                                 continue;
3127
3128                         kthread_unpark(adev->rings[i]->sched.thread);
3129                 }
3130         }
3131
3132         if (amdgpu_device_has_dc_support(adev)) {
3133                 if (drm_atomic_helper_resume(adev->ddev, state))
3134                         dev_info(adev->dev, "drm resume failed:%d\n", r);
3135                 amdgpu_dm_display_resume(adev);
3136         } else {
3137                 drm_helper_resume_force_mode(adev->ddev);
3138         }
3139
3140         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
3141
3142         if (r) {
3143                 /* bad news, how to tell it to userspace ? */
3144                 dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
3145                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
3146         } else {
3147                 dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
3148         }
3149
3150         amdgpu_vf_error_trans_all(adev);
3151         adev->in_gpu_reset = 0;
3152         mutex_unlock(&adev->lock_reset);
3153         return r;
3154 }
3155
3156 void amdgpu_get_pcie_info(struct amdgpu_device *adev)
3157 {
3158         u32 mask;
3159         int ret;
3160
3161         if (amdgpu_pcie_gen_cap)
3162                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
3163
3164         if (amdgpu_pcie_lane_cap)
3165                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
3166
3167         /* covers APUs as well */
3168         if (pci_is_root_bus(adev->pdev->bus)) {
3169                 if (adev->pm.pcie_gen_mask == 0)
3170                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3171                 if (adev->pm.pcie_mlw_mask == 0)
3172                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3173                 return;
3174         }
3175
3176         if (adev->pm.pcie_gen_mask == 0) {
3177                 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
3178                 if (!ret) {
3179                         adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3180                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3181                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
3182
3183                         if (mask & DRM_PCIE_SPEED_25)
3184                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
3185                         if (mask & DRM_PCIE_SPEED_50)
3186                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
3187                         if (mask & DRM_PCIE_SPEED_80)
3188                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
3189                 } else {
3190                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3191                 }
3192         }
3193         if (adev->pm.pcie_mlw_mask == 0) {
3194                 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
3195                 if (!ret) {
3196                         switch (mask) {
3197                         case 32:
3198                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
3199                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3200                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3201                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3202                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3203                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3204                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3205                                 break;
3206                         case 16:
3207                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3208                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3209                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3210                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3211                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3212                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3213                                 break;
3214                         case 12:
3215                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3216                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3217                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3218                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3219                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3220                                 break;
3221                         case 8:
3222                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3223                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3224                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3225                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3226                                 break;
3227                         case 4:
3228                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3229                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3230                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3231                                 break;
3232                         case 2:
3233                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3234                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3235                                 break;
3236                         case 1:
3237                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
3238                                 break;
3239                         default:
3240                                 break;
3241                         }
3242                 } else {
3243                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3244                 }
3245         }
3246 }
3247
3248 /*
3249  * Debugfs
3250  */
3251 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
3252                              const struct drm_info_list *files,
3253                              unsigned nfiles)
3254 {
3255         unsigned i;
3256
3257         for (i = 0; i < adev->debugfs_count; i++) {
3258                 if (adev->debugfs[i].files == files) {
3259                         /* Already registered */
3260                         return 0;
3261                 }
3262         }
3263
3264         i = adev->debugfs_count + 1;
3265         if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
3266                 DRM_ERROR("Reached maximum number of debugfs components.\n");
3267                 DRM_ERROR("Report so we increase "
3268                           "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
3269                 return -EINVAL;
3270         }
3271         adev->debugfs[adev->debugfs_count].files = files;
3272         adev->debugfs[adev->debugfs_count].num_files = nfiles;
3273         adev->debugfs_count = i;
3274 #if defined(CONFIG_DEBUG_FS)
3275         drm_debugfs_create_files(files, nfiles,
3276                                  adev->ddev->primary->debugfs_root,
3277                                  adev->ddev->primary);
3278 #endif
3279         return 0;
3280 }
3281
3282 #if defined(CONFIG_DEBUG_FS)
3283
3284 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
3285                                         size_t size, loff_t *pos)
3286 {
3287         struct amdgpu_device *adev = file_inode(f)->i_private;
3288         ssize_t result = 0;
3289         int r;
3290         bool pm_pg_lock, use_bank;
3291         unsigned instance_bank, sh_bank, se_bank;
3292
3293         if (size & 0x3 || *pos & 0x3)
3294                 return -EINVAL;
3295
3296         /* are we reading registers for which a PG lock is necessary? */
3297         pm_pg_lock = (*pos >> 23) & 1;
3298
3299         if (*pos & (1ULL << 62)) {
3300                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
3301                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
3302                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
3303
3304                 if (se_bank == 0x3FF)
3305                         se_bank = 0xFFFFFFFF;
3306                 if (sh_bank == 0x3FF)
3307                         sh_bank = 0xFFFFFFFF;
3308                 if (instance_bank == 0x3FF)
3309                         instance_bank = 0xFFFFFFFF;
3310                 use_bank = 1;
3311         } else {
3312                 use_bank = 0;
3313         }
3314
3315         *pos &= (1UL << 22) - 1;
3316
3317         if (use_bank) {
3318                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
3319                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
3320                         return -EINVAL;
3321                 mutex_lock(&adev->grbm_idx_mutex);
3322                 amdgpu_gfx_select_se_sh(adev, se_bank,
3323                                         sh_bank, instance_bank);
3324         }
3325
3326         if (pm_pg_lock)
3327                 mutex_lock(&adev->pm.mutex);
3328
3329         while (size) {
3330                 uint32_t value;
3331
3332                 if (*pos > adev->rmmio_size)
3333                         goto end;
3334
3335                 value = RREG32(*pos >> 2);
3336                 r = put_user(value, (uint32_t *)buf);
3337                 if (r) {
3338                         result = r;
3339                         goto end;
3340                 }
3341
3342                 result += 4;
3343                 buf += 4;
3344                 *pos += 4;
3345                 size -= 4;
3346         }
3347
3348 end:
3349         if (use_bank) {
3350                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3351                 mutex_unlock(&adev->grbm_idx_mutex);
3352         }
3353
3354         if (pm_pg_lock)
3355                 mutex_unlock(&adev->pm.mutex);
3356
3357         return result;
3358 }
3359
3360 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
3361                                          size_t size, loff_t *pos)
3362 {
3363         struct amdgpu_device *adev = file_inode(f)->i_private;
3364         ssize_t result = 0;
3365         int r;
3366         bool pm_pg_lock, use_bank;
3367         unsigned instance_bank, sh_bank, se_bank;
3368
3369         if (size & 0x3 || *pos & 0x3)
3370                 return -EINVAL;
3371
3372         /* are we reading registers for which a PG lock is necessary? */
3373         pm_pg_lock = (*pos >> 23) & 1;
3374
3375         if (*pos & (1ULL << 62)) {
3376                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
3377                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
3378                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
3379
3380                 if (se_bank == 0x3FF)
3381                         se_bank = 0xFFFFFFFF;
3382                 if (sh_bank == 0x3FF)
3383                         sh_bank = 0xFFFFFFFF;
3384                 if (instance_bank == 0x3FF)
3385                         instance_bank = 0xFFFFFFFF;
3386                 use_bank = 1;
3387         } else {
3388                 use_bank = 0;
3389         }
3390
3391         *pos &= (1UL << 22) - 1;
3392
3393         if (use_bank) {
3394                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
3395                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
3396                         return -EINVAL;
3397                 mutex_lock(&adev->grbm_idx_mutex);
3398                 amdgpu_gfx_select_se_sh(adev, se_bank,
3399                                         sh_bank, instance_bank);
3400         }
3401
3402         if (pm_pg_lock)
3403                 mutex_lock(&adev->pm.mutex);
3404
3405         while (size) {
3406                 uint32_t value;
3407
3408                 if (*pos > adev->rmmio_size)
3409                         return result;
3410
3411                 r = get_user(value, (uint32_t *)buf);
3412                 if (r)
3413                         return r;
3414
3415                 WREG32(*pos >> 2, value);
3416
3417                 result += 4;
3418                 buf += 4;
3419                 *pos += 4;
3420                 size -= 4;
3421         }
3422
3423         if (use_bank) {
3424                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3425                 mutex_unlock(&adev->grbm_idx_mutex);
3426         }
3427
3428         if (pm_pg_lock)
3429                 mutex_unlock(&adev->pm.mutex);
3430
3431         return result;
3432 }
3433
3434 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
3435                                         size_t size, loff_t *pos)
3436 {
3437         struct amdgpu_device *adev = file_inode(f)->i_private;
3438         ssize_t result = 0;
3439         int r;
3440
3441         if (size & 0x3 || *pos & 0x3)
3442                 return -EINVAL;
3443
3444         while (size) {
3445                 uint32_t value;
3446
3447                 value = RREG32_PCIE(*pos >> 2);
3448                 r = put_user(value, (uint32_t *)buf);
3449                 if (r)
3450                         return r;
3451
3452                 result += 4;
3453                 buf += 4;
3454                 *pos += 4;
3455                 size -= 4;
3456         }
3457
3458         return result;
3459 }
3460
3461 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
3462                                          size_t size, loff_t *pos)
3463 {
3464         struct amdgpu_device *adev = file_inode(f)->i_private;
3465         ssize_t result = 0;
3466         int r;
3467
3468         if (size & 0x3 || *pos & 0x3)
3469                 return -EINVAL;
3470
3471         while (size) {
3472                 uint32_t value;
3473
3474                 r = get_user(value, (uint32_t *)buf);
3475                 if (r)
3476                         return r;
3477
3478                 WREG32_PCIE(*pos >> 2, value);
3479
3480                 result += 4;
3481                 buf += 4;
3482                 *pos += 4;
3483                 size -= 4;
3484         }
3485
3486         return result;
3487 }
3488
3489 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
3490                                         size_t size, loff_t *pos)
3491 {
3492         struct amdgpu_device *adev = file_inode(f)->i_private;
3493         ssize_t result = 0;
3494         int r;
3495
3496         if (size & 0x3 || *pos & 0x3)
3497                 return -EINVAL;
3498
3499         while (size) {
3500                 uint32_t value;
3501
3502                 value = RREG32_DIDT(*pos >> 2);
3503                 r = put_user(value, (uint32_t *)buf);
3504                 if (r)
3505                         return r;
3506
3507                 result += 4;
3508                 buf += 4;
3509                 *pos += 4;
3510                 size -= 4;
3511         }
3512
3513         return result;
3514 }
3515
3516 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
3517                                          size_t size, loff_t *pos)
3518 {
3519         struct amdgpu_device *adev = file_inode(f)->i_private;
3520         ssize_t result = 0;
3521         int r;
3522
3523         if (size & 0x3 || *pos & 0x3)
3524                 return -EINVAL;
3525
3526         while (size) {
3527                 uint32_t value;
3528
3529                 r = get_user(value, (uint32_t *)buf);
3530                 if (r)
3531                         return r;
3532
3533                 WREG32_DIDT(*pos >> 2, value);
3534
3535                 result += 4;
3536                 buf += 4;
3537                 *pos += 4;
3538                 size -= 4;
3539         }
3540
3541         return result;
3542 }
3543
3544 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
3545                                         size_t size, loff_t *pos)
3546 {
3547         struct amdgpu_device *adev = file_inode(f)->i_private;
3548         ssize_t result = 0;
3549         int r;
3550
3551         if (size & 0x3 || *pos & 0x3)
3552                 return -EINVAL;
3553
3554         while (size) {
3555                 uint32_t value;
3556
3557                 value = RREG32_SMC(*pos);
3558                 r = put_user(value, (uint32_t *)buf);
3559                 if (r)
3560                         return r;
3561
3562                 result += 4;
3563                 buf += 4;
3564                 *pos += 4;
3565                 size -= 4;
3566         }
3567
3568         return result;
3569 }
3570
3571 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
3572                                          size_t size, loff_t *pos)
3573 {
3574         struct amdgpu_device *adev = file_inode(f)->i_private;
3575         ssize_t result = 0;
3576         int r;
3577
3578         if (size & 0x3 || *pos & 0x3)
3579                 return -EINVAL;
3580
3581         while (size) {
3582                 uint32_t value;
3583
3584                 r = get_user(value, (uint32_t *)buf);
3585                 if (r)
3586                         return r;
3587
3588                 WREG32_SMC(*pos, value);
3589
3590                 result += 4;
3591                 buf += 4;
3592                 *pos += 4;
3593                 size -= 4;
3594         }
3595
3596         return result;
3597 }
3598
3599 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
3600                                         size_t size, loff_t *pos)
3601 {
3602         struct amdgpu_device *adev = file_inode(f)->i_private;
3603         ssize_t result = 0;
3604         int r;
3605         uint32_t *config, no_regs = 0;
3606
3607         if (size & 0x3 || *pos & 0x3)
3608                 return -EINVAL;
3609
3610         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
3611         if (!config)
3612                 return -ENOMEM;
3613
3614         /* version, increment each time something is added */
3615         config[no_regs++] = 3;
3616         config[no_regs++] = adev->gfx.config.max_shader_engines;
3617         config[no_regs++] = adev->gfx.config.max_tile_pipes;
3618         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
3619         config[no_regs++] = adev->gfx.config.max_sh_per_se;
3620         config[no_regs++] = adev->gfx.config.max_backends_per_se;
3621         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
3622         config[no_regs++] = adev->gfx.config.max_gprs;
3623         config[no_regs++] = adev->gfx.config.max_gs_threads;
3624         config[no_regs++] = adev->gfx.config.max_hw_contexts;
3625         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
3626         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
3627         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
3628         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
3629         config[no_regs++] = adev->gfx.config.num_tile_pipes;
3630         config[no_regs++] = adev->gfx.config.backend_enable_mask;
3631         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
3632         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
3633         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
3634         config[no_regs++] = adev->gfx.config.num_gpus;
3635         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
3636         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
3637         config[no_regs++] = adev->gfx.config.gb_addr_config;
3638         config[no_regs++] = adev->gfx.config.num_rbs;
3639
3640         /* rev==1 */
3641         config[no_regs++] = adev->rev_id;
3642         config[no_regs++] = adev->pg_flags;
3643         config[no_regs++] = adev->cg_flags;
3644
3645         /* rev==2 */
3646         config[no_regs++] = adev->family;
3647         config[no_regs++] = adev->external_rev_id;
3648
3649         /* rev==3 */
3650         config[no_regs++] = adev->pdev->device;
3651         config[no_regs++] = adev->pdev->revision;
3652         config[no_regs++] = adev->pdev->subsystem_device;
3653         config[no_regs++] = adev->pdev->subsystem_vendor;
3654
3655         while (size && (*pos < no_regs * 4)) {
3656                 uint32_t value;
3657
3658                 value = config[*pos >> 2];
3659                 r = put_user(value, (uint32_t *)buf);
3660                 if (r) {
3661                         kfree(config);
3662                         return r;
3663                 }
3664
3665                 result += 4;
3666                 buf += 4;
3667                 *pos += 4;
3668                 size -= 4;
3669         }
3670
3671         kfree(config);
3672         return result;
3673 }
3674
3675 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
3676                                         size_t size, loff_t *pos)
3677 {
3678         struct amdgpu_device *adev = file_inode(f)->i_private;
3679         int idx, x, outsize, r, valuesize;
3680         uint32_t values[16];
3681
3682         if (size & 3 || *pos & 0x3)
3683                 return -EINVAL;
3684
3685         if (amdgpu_dpm == 0)
3686                 return -EINVAL;
3687
3688         /* convert offset to sensor number */
3689         idx = *pos >> 2;
3690
3691         valuesize = sizeof(values);
3692         if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
3693                 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
3694         else
3695                 return -EINVAL;
3696
3697         if (size > valuesize)
3698                 return -EINVAL;
3699
3700         outsize = 0;
3701         x = 0;
3702         if (!r) {
3703                 while (size) {
3704                         r = put_user(values[x++], (int32_t *)buf);
3705                         buf += 4;
3706                         size -= 4;
3707                         outsize += 4;
3708                 }
3709         }
3710
3711         return !r ? outsize : r;
3712 }
3713
3714 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
3715                                         size_t size, loff_t *pos)
3716 {
3717         struct amdgpu_device *adev = f->f_inode->i_private;
3718         int r, x;
3719         ssize_t result=0;
3720         uint32_t offset, se, sh, cu, wave, simd, data[32];
3721
3722         if (size & 3 || *pos & 3)
3723                 return -EINVAL;
3724
3725         /* decode offset */
3726         offset = (*pos & GENMASK_ULL(6, 0));
3727         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
3728         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
3729         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
3730         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
3731         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
3732
3733         /* switch to the specific se/sh/cu */
3734         mutex_lock(&adev->grbm_idx_mutex);
3735         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3736
3737         x = 0;
3738         if (adev->gfx.funcs->read_wave_data)
3739                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
3740
3741         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3742         mutex_unlock(&adev->grbm_idx_mutex);
3743
3744         if (!x)
3745                 return -EINVAL;
3746
3747         while (size && (offset < x * 4)) {
3748                 uint32_t value;
3749
3750                 value = data[offset >> 2];
3751                 r = put_user(value, (uint32_t *)buf);
3752                 if (r)
3753                         return r;
3754
3755                 result += 4;
3756                 buf += 4;
3757                 offset += 4;
3758                 size -= 4;
3759         }
3760
3761         return result;
3762 }
3763
3764 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
3765                                         size_t size, loff_t *pos)
3766 {
3767         struct amdgpu_device *adev = f->f_inode->i_private;
3768         int r;
3769         ssize_t result = 0;
3770         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
3771
3772         if (size & 3 || *pos & 3)
3773                 return -EINVAL;
3774
3775         /* decode offset */
3776         offset = *pos & GENMASK_ULL(11, 0);
3777         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
3778         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
3779         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
3780         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
3781         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
3782         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
3783         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
3784
3785         data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
3786         if (!data)
3787                 return -ENOMEM;
3788
3789         /* switch to the specific se/sh/cu */
3790         mutex_lock(&adev->grbm_idx_mutex);
3791         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3792
3793         if (bank == 0) {
3794                 if (adev->gfx.funcs->read_wave_vgprs)
3795                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
3796         } else {
3797                 if (adev->gfx.funcs->read_wave_sgprs)
3798                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
3799         }
3800
3801         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3802         mutex_unlock(&adev->grbm_idx_mutex);
3803
3804         while (size) {
3805                 uint32_t value;
3806
3807                 value = data[offset++];
3808                 r = put_user(value, (uint32_t *)buf);
3809                 if (r) {
3810                         result = r;
3811                         goto err;
3812                 }
3813
3814                 result += 4;
3815                 buf += 4;
3816                 size -= 4;
3817         }
3818
3819 err:
3820         kfree(data);
3821         return result;
3822 }
3823
3824 static const struct file_operations amdgpu_debugfs_regs_fops = {
3825         .owner = THIS_MODULE,
3826         .read = amdgpu_debugfs_regs_read,
3827         .write = amdgpu_debugfs_regs_write,
3828         .llseek = default_llseek
3829 };
3830 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
3831         .owner = THIS_MODULE,
3832         .read = amdgpu_debugfs_regs_didt_read,
3833         .write = amdgpu_debugfs_regs_didt_write,
3834         .llseek = default_llseek
3835 };
3836 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
3837         .owner = THIS_MODULE,
3838         .read = amdgpu_debugfs_regs_pcie_read,
3839         .write = amdgpu_debugfs_regs_pcie_write,
3840         .llseek = default_llseek
3841 };
3842 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
3843         .owner = THIS_MODULE,
3844         .read = amdgpu_debugfs_regs_smc_read,
3845         .write = amdgpu_debugfs_regs_smc_write,
3846         .llseek = default_llseek
3847 };
3848
3849 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
3850         .owner = THIS_MODULE,
3851         .read = amdgpu_debugfs_gca_config_read,
3852         .llseek = default_llseek
3853 };
3854
3855 static const struct file_operations amdgpu_debugfs_sensors_fops = {
3856         .owner = THIS_MODULE,
3857         .read = amdgpu_debugfs_sensor_read,
3858         .llseek = default_llseek
3859 };
3860
3861 static const struct file_operations amdgpu_debugfs_wave_fops = {
3862         .owner = THIS_MODULE,
3863         .read = amdgpu_debugfs_wave_read,
3864         .llseek = default_llseek
3865 };
3866 static const struct file_operations amdgpu_debugfs_gpr_fops = {
3867         .owner = THIS_MODULE,
3868         .read = amdgpu_debugfs_gpr_read,
3869         .llseek = default_llseek
3870 };
3871
3872 static const struct file_operations *debugfs_regs[] = {
3873         &amdgpu_debugfs_regs_fops,
3874         &amdgpu_debugfs_regs_didt_fops,
3875         &amdgpu_debugfs_regs_pcie_fops,
3876         &amdgpu_debugfs_regs_smc_fops,
3877         &amdgpu_debugfs_gca_config_fops,
3878         &amdgpu_debugfs_sensors_fops,
3879         &amdgpu_debugfs_wave_fops,
3880         &amdgpu_debugfs_gpr_fops,
3881 };
3882
3883 static const char *debugfs_regs_names[] = {
3884         "amdgpu_regs",
3885         "amdgpu_regs_didt",
3886         "amdgpu_regs_pcie",
3887         "amdgpu_regs_smc",
3888         "amdgpu_gca_config",
3889         "amdgpu_sensors",
3890         "amdgpu_wave",
3891         "amdgpu_gpr",
3892 };
3893
3894 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3895 {
3896         struct drm_minor *minor = adev->ddev->primary;
3897         struct dentry *ent, *root = minor->debugfs_root;
3898         unsigned i, j;
3899
3900         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3901                 ent = debugfs_create_file(debugfs_regs_names[i],
3902                                           S_IFREG | S_IRUGO, root,
3903                                           adev, debugfs_regs[i]);
3904                 if (IS_ERR(ent)) {
3905                         for (j = 0; j < i; j++) {
3906                                 debugfs_remove(adev->debugfs_regs[i]);
3907                                 adev->debugfs_regs[i] = NULL;
3908                         }
3909                         return PTR_ERR(ent);
3910                 }
3911
3912                 if (!i)
3913                         i_size_write(ent->d_inode, adev->rmmio_size);
3914                 adev->debugfs_regs[i] = ent;
3915         }
3916
3917         return 0;
3918 }
3919
3920 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
3921 {
3922         unsigned i;
3923
3924         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3925                 if (adev->debugfs_regs[i]) {
3926                         debugfs_remove(adev->debugfs_regs[i]);
3927                         adev->debugfs_regs[i] = NULL;
3928                 }
3929         }
3930 }
3931
3932 static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
3933 {
3934         struct drm_info_node *node = (struct drm_info_node *) m->private;
3935         struct drm_device *dev = node->minor->dev;
3936         struct amdgpu_device *adev = dev->dev_private;
3937         int r = 0, i;
3938
3939         /* hold on the scheduler */
3940         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
3941                 struct amdgpu_ring *ring = adev->rings[i];
3942
3943                 if (!ring || !ring->sched.thread)
3944                         continue;
3945                 kthread_park(ring->sched.thread);
3946         }
3947
3948         seq_printf(m, "run ib test:\n");
3949         r = amdgpu_ib_ring_tests(adev);
3950         if (r)
3951                 seq_printf(m, "ib ring tests failed (%d).\n", r);
3952         else
3953                 seq_printf(m, "ib ring tests passed.\n");
3954
3955         /* go on the scheduler */
3956         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
3957                 struct amdgpu_ring *ring = adev->rings[i];
3958
3959                 if (!ring || !ring->sched.thread)
3960                         continue;
3961                 kthread_unpark(ring->sched.thread);
3962         }
3963
3964         return 0;
3965 }
3966
3967 static const struct drm_info_list amdgpu_debugfs_test_ib_ring_list[] = {
3968         {"amdgpu_test_ib", &amdgpu_debugfs_test_ib}
3969 };
3970
3971 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
3972 {
3973         return amdgpu_debugfs_add_files(adev,
3974                                         amdgpu_debugfs_test_ib_ring_list, 1);
3975 }
3976
3977 int amdgpu_debugfs_init(struct drm_minor *minor)
3978 {
3979         return 0;
3980 }
3981
3982 static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
3983 {
3984         struct drm_info_node *node = (struct drm_info_node *) m->private;
3985         struct drm_device *dev = node->minor->dev;
3986         struct amdgpu_device *adev = dev->dev_private;
3987
3988         seq_write(m, adev->bios, adev->bios_size);
3989         return 0;
3990 }
3991
3992 static const struct drm_info_list amdgpu_vbios_dump_list[] = {
3993                 {"amdgpu_vbios",
3994                  amdgpu_debugfs_get_vbios_dump,
3995                  0, NULL},
3996 };
3997
3998 static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
3999 {
4000         return amdgpu_debugfs_add_files(adev,
4001                                         amdgpu_vbios_dump_list, 1);
4002 }
4003 #else
4004 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
4005 {
4006         return 0;
4007 }
4008 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
4009 {
4010         return 0;
4011 }
4012 static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
4013 {
4014         return 0;
4015 }
4016 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
4017 #endif