drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/console.h>
  31 #include <linux/slab.h>
  32 #include <drm/drmP.h>
  33 #include <drm/drm_atomic_helper.h>
  34 #include <drm/drm_probe_helper.h>
  35 #include <drm/amdgpu_drm.h>
  36 #include <linux/vgaarb.h>
  37 #include <linux/vga_switcheroo.h>
  38 #include <linux/efi.h>
  39 #include "amdgpu.h"
  40 #include "amdgpu_trace.h"
  41 #include "amdgpu_i2c.h"
  42 #include "atom.h"
  43 #include "amdgpu_atombios.h"
  44 #include "amdgpu_atomfirmware.h"
  45 #include "amd_pcie.h"
  46 #ifdef CONFIG_DRM_AMDGPU_SI
  47 #include "si.h"
  48 #endif
  49 #ifdef CONFIG_DRM_AMDGPU_CIK
  50 #include "cik.h"
  51 #endif
  52 #include "vi.h"
  53 #include "soc15.h"
  54 #include "bif/bif_4_1_d.h"
  55 #include <linux/pci.h>
  56 #include <linux/firmware.h>
  57 #include "amdgpu_vf_error.h"
  58
  59 #include "amdgpu_amdkfd.h"
  60 #include "amdgpu_pm.h"
  61
  62 #include "amdgpu_xgmi.h"
  63 #include "amdgpu_ras.h"
  64 #include "amdgpu_pmu.h"
  65
  66 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  67 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  68 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  69 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  70 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  71 MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
  72
  73 #define AMDGPU_RESUME_MS                2000
  74
  75 static const char *amdgpu_asic_name[] = {
  76         "TAHITI",
  77         "PITCAIRN",
  78         "VERDE",
  79         "OLAND",
  80         "HAINAN",
  81         "BONAIRE",
  82         "KAVERI",
  83         "KABINI",
  84         "HAWAII",
  85         "MULLINS",
  86         "TOPAZ",
  87         "TONGA",
  88         "FIJI",
  89         "CARRIZO",
  90         "STONEY",
  91         "POLARIS10",
  92         "POLARIS11",
  93         "POLARIS12",
  94         "VEGAM",
  95         "VEGA10",
  96         "VEGA12",
  97         "VEGA20",
  98         "RAVEN",
  99         "NAVI10",
 100         "LAST",
 101 };
 102
 103 /**
 104  * DOC: pcie_replay_count
 105  *
 106  * The amdgpu driver provides a sysfs API for reporting the total number
 107  * of PCIe replays (NAKs)
 108  * The file pcie_replay_count is used for this and returns the total
 109  * number of replays as a sum of the NAKs generated and NAKs received
 110  */
 111
 112 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 113                 struct device_attribute *attr, char *buf)
 114 {
 115         struct drm_device *ddev = dev_get_drvdata(dev);
 116         struct amdgpu_device *adev = ddev->dev_private;
 117         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 118
 119         return snprintf(buf, PAGE_SIZE, "%llu\n", cnt);
 120 }
 121
 122 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
 123                 amdgpu_device_get_pcie_replay_count, NULL);
 124
 125 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 126
 127 /**
 128  * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control
 129  *
 130  * @dev: drm_device pointer
 131  *
 132  * Returns true if the device is a dGPU with HG/PX power control,
 133  * otherwise return false.
 134  */
 135 bool amdgpu_device_is_px(struct drm_device *dev)
 136 {
 137         struct amdgpu_device *adev = dev->dev_private;
 138
 139         if (adev->flags & AMD_IS_PX)
 140                 return true;
 141         return false;
 142 }
 143
 144 /*
 145  * MMIO register access helper functions.
 146  */
 147 /**
 148  * amdgpu_mm_rreg - read a memory mapped IO register
 149  *
 150  * @adev: amdgpu_device pointer
 151  * @reg: dword aligned register offset
 152  * @acc_flags: access flags which require special behavior
 153  *
 154  * Returns the 32 bit value from the offset specified.
 155  */
 156 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
 157                         uint32_t acc_flags)
 158 {
 159         uint32_t ret;
 160
 161         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
 162                 return amdgpu_virt_kiq_rreg(adev, reg);
 163
 164         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 165                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 166         else {
 167                 unsigned long flags;
 168
 169                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 170                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 171                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 172                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 173         }
 174         trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
 175         return ret;
 176 }
 177
 178 /*
 179  * MMIO register read with bytes helper functions
 180  * @offset:bytes offset from MMIO start
 181  *
 182 */
 183
 184 /**
 185  * amdgpu_mm_rreg8 - read a memory mapped IO register
 186  *
 187  * @adev: amdgpu_device pointer
 188  * @offset: byte aligned register offset
 189  *
 190  * Returns the 8 bit value from the offset specified.
 191  */
 192 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
 193         if (offset < adev->rmmio_size)
 194                 return (readb(adev->rmmio + offset));
 195         BUG();
 196 }
 197
 198 /*
 199  * MMIO register write with bytes helper functions
 200  * @offset:bytes offset from MMIO start
 201  * @value: the value want to be written to the register
 202  *
 203 */
 204 /**
 205  * amdgpu_mm_wreg8 - read a memory mapped IO register
 206  *
 207  * @adev: amdgpu_device pointer
 208  * @offset: byte aligned register offset
 209  * @value: 8 bit value to write
 210  *
 211  * Writes the value specified to the offset specified.
 212  */
 213 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
 214         if (offset < adev->rmmio_size)
 215                 writeb(value, adev->rmmio + offset);
 216         else
 217                 BUG();
 218 }
 219
 220 /**
 221  * amdgpu_mm_wreg - write to a memory mapped IO register
 222  *
 223  * @adev: amdgpu_device pointer
 224  * @reg: dword aligned register offset
 225  * @v: 32 bit value to write to the register
 226  * @acc_flags: access flags which require special behavior
 227  *
 228  * Writes the value specified to the offset specified.
 229  */
 230 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 231                     uint32_t acc_flags)
 232 {
 233         trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 234
 235         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
 236                 adev->last_mm_index = v;
 237         }
 238
 239         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
 240                 return amdgpu_virt_kiq_wreg(adev, reg, v);
 241
 242         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 243                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 244         else {
 245                 unsigned long flags;
 246
 247                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 248                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 249                 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 250                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 251         }
 252
 253         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
 254                 udelay(500);
 255         }
 256 }
 257
 258 /**
 259  * amdgpu_io_rreg - read an IO register
 260  *
 261  * @adev: amdgpu_device pointer
 262  * @reg: dword aligned register offset
 263  *
 264  * Returns the 32 bit value from the offset specified.
 265  */
 266 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
 267 {
 268         if ((reg * 4) < adev->rio_mem_size)
 269                 return ioread32(adev->rio_mem + (reg * 4));
 270         else {
 271                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 272                 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
 273         }
 274 }
 275
 276 /**
 277  * amdgpu_io_wreg - write to an IO register
 278  *
 279  * @adev: amdgpu_device pointer
 280  * @reg: dword aligned register offset
 281  * @v: 32 bit value to write to the register
 282  *
 283  * Writes the value specified to the offset specified.
 284  */
 285 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 286 {
 287         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
 288                 adev->last_mm_index = v;
 289         }
 290
 291         if ((reg * 4) < adev->rio_mem_size)
 292                 iowrite32(v, adev->rio_mem + (reg * 4));
 293         else {
 294                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 295                 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
 296         }
 297
 298         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
 299                 udelay(500);
 300         }
 301 }
 302
 303 /**
 304  * amdgpu_mm_rdoorbell - read a doorbell dword
 305  *
 306  * @adev: amdgpu_device pointer
 307  * @index: doorbell index
 308  *
 309  * Returns the value in the doorbell aperture at the
 310  * requested doorbell index (CIK).
 311  */
 312 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 313 {
 314         if (index < adev->doorbell.num_doorbells) {
 315                 return readl(adev->doorbell.ptr + index);
 316         } else {
 317                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 318                 return 0;
 319         }
 320 }
 321
 322 /**
 323  * amdgpu_mm_wdoorbell - write a doorbell dword
 324  *
 325  * @adev: amdgpu_device pointer
 326  * @index: doorbell index
 327  * @v: value to write
 328  *
 329  * Writes @v to the doorbell aperture at the
 330  * requested doorbell index (CIK).
 331  */
 332 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 333 {
 334         if (index < adev->doorbell.num_doorbells) {
 335                 writel(v, adev->doorbell.ptr + index);
 336         } else {
 337                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 338         }
 339 }
 340
 341 /**
 342  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 343  *
 344  * @adev: amdgpu_device pointer
 345  * @index: doorbell index
 346  *
 347  * Returns the value in the doorbell aperture at the
 348  * requested doorbell index (VEGA10+).
 349  */
 350 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 351 {
 352         if (index < adev->doorbell.num_doorbells) {
 353                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 354         } else {
 355                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 356                 return 0;
 357         }
 358 }
 359
 360 /**
 361  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 362  *
 363  * @adev: amdgpu_device pointer
 364  * @index: doorbell index
 365  * @v: value to write
 366  *
 367  * Writes @v to the doorbell aperture at the
 368  * requested doorbell index (VEGA10+).
 369  */
 370 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 371 {
 372         if (index < adev->doorbell.num_doorbells) {
 373                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 374         } else {
 375                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 376         }
 377 }
 378
 379 /**
 380  * amdgpu_invalid_rreg - dummy reg read function
 381  *
 382  * @adev: amdgpu device pointer
 383  * @reg: offset of register
 384  *
 385  * Dummy register read function.  Used for register blocks
 386  * that certain asics don't have (all asics).
 387  * Returns the value in the register.
 388  */
 389 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 390 {
 391         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 392         BUG();
 393         return 0;
 394 }
 395
 396 /**
 397  * amdgpu_invalid_wreg - dummy reg write function
 398  *
 399  * @adev: amdgpu device pointer
 400  * @reg: offset of register
 401  * @v: value to write to the register
 402  *
 403  * Dummy register read function.  Used for register blocks
 404  * that certain asics don't have (all asics).
 405  */
 406 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 407 {
 408         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 409                   reg, v);
 410         BUG();
 411 }
 412
 413 /**
 414  * amdgpu_block_invalid_rreg - dummy reg read function
 415  *
 416  * @adev: amdgpu device pointer
 417  * @block: offset of instance
 418  * @reg: offset of register
 419  *
 420  * Dummy register read function.  Used for register blocks
 421  * that certain asics don't have (all asics).
 422  * Returns the value in the register.
 423  */
 424 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 425                                           uint32_t block, uint32_t reg)
 426 {
 427         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 428                   reg, block);
 429         BUG();
 430         return 0;
 431 }
 432
 433 /**
 434  * amdgpu_block_invalid_wreg - dummy reg write function
 435  *
 436  * @adev: amdgpu device pointer
 437  * @block: offset of instance
 438  * @reg: offset of register
 439  * @v: value to write to the register
 440  *
 441  * Dummy register read function.  Used for register blocks
 442  * that certain asics don't have (all asics).
 443  */
 444 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 445                                       uint32_t block,
 446                                       uint32_t reg, uint32_t v)
 447 {
 448         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 449                   reg, block, v);
 450         BUG();
 451 }
 452
 453 /**
 454  * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
 455  *
 456  * @adev: amdgpu device pointer
 457  *
 458  * Allocates a scratch page of VRAM for use by various things in the
 459  * driver.
 460  */
 461 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
 462 {
 463         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 464                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 465                                        &adev->vram_scratch.robj,
 466                                        &adev->vram_scratch.gpu_addr,
 467                                        (void **)&adev->vram_scratch.ptr);
 468 }
 469
 470 /**
 471  * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
 472  *
 473  * @adev: amdgpu device pointer
 474  *
 475  * Frees the VRAM scratch page.
 476  */
 477 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
 478 {
 479         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 480 }
 481
 482 /**
 483  * amdgpu_device_program_register_sequence - program an array of registers.
 484  *
 485  * @adev: amdgpu_device pointer
 486  * @registers: pointer to the register array
 487  * @array_size: size of the register array
 488  *
 489  * Programs an array or registers with and and or masks.
 490  * This is a helper for setting golden registers.
 491  */
 492 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 493                                              const u32 *registers,
 494                                              const u32 array_size)
 495 {
 496         u32 tmp, reg, and_mask, or_mask;
 497         int i;
 498
 499         if (array_size % 3)
 500                 return;
 501
 502         for (i = 0; i < array_size; i +=3) {
 503                 reg = registers[i + 0];
 504                 and_mask = registers[i + 1];
 505                 or_mask = registers[i + 2];
 506
 507                 if (and_mask == 0xffffffff) {
 508                         tmp = or_mask;
 509                 } else {
 510                         tmp = RREG32(reg);
 511                         tmp &= ~and_mask;
 512                         tmp |= or_mask;
 513                 }
 514                 WREG32(reg, tmp);
 515         }
 516 }
 517
 518 /**
 519  * amdgpu_device_pci_config_reset - reset the GPU
 520  *
 521  * @adev: amdgpu_device pointer
 522  *
 523  * Resets the GPU using the pci config reset sequence.
 524  * Only applicable to asics prior to vega10.
 525  */
 526 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
 527 {
 528         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 529 }
 530
 531 /*
 532  * GPU doorbell aperture helpers function.
 533  */
 534 /**
 535  * amdgpu_device_doorbell_init - Init doorbell driver information.
 536  *
 537  * @adev: amdgpu_device pointer
 538  *
 539  * Init doorbell driver information (CIK)
 540  * Returns 0 on success, error on failure.
 541  */
 542 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
 543 {
 544
 545         /* No doorbell on SI hardware generation */
 546         if (adev->asic_type < CHIP_BONAIRE) {
 547                 adev->doorbell.base = 0;
 548                 adev->doorbell.size = 0;
 549                 adev->doorbell.num_doorbells = 0;
 550                 adev->doorbell.ptr = NULL;
 551                 return 0;
 552         }
 553
 554         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
 555                 return -EINVAL;
 556
 557         amdgpu_asic_init_doorbell_index(adev);
 558
 559         /* doorbell bar mapping */
 560         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 561         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 562
 563         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 564                                              adev->doorbell_index.max_assignment+1);
 565         if (adev->doorbell.num_doorbells == 0)
 566                 return -EINVAL;
 567
 568         /* For Vega, reserve and map two pages on doorbell BAR since SDMA
 569          * paging queue doorbell use the second page. The
 570          * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
 571          * doorbells are in the first page. So with paging queue enabled,
 572          * the max num_doorbells should + 1 page (0x400 in dword)
 573          */
 574         if (adev->asic_type >= CHIP_VEGA10)
 575                 adev->doorbell.num_doorbells += 0x400;
 576
 577         adev->doorbell.ptr = ioremap(adev->doorbell.base,
 578                                      adev->doorbell.num_doorbells *
 579                                      sizeof(u32));
 580         if (adev->doorbell.ptr == NULL)
 581                 return -ENOMEM;
 582
 583         return 0;
 584 }
 585
 586 /**
 587  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
 588  *
 589  * @adev: amdgpu_device pointer
 590  *
 591  * Tear down doorbell driver information (CIK)
 592  */
 593 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
 594 {
 595         iounmap(adev->doorbell.ptr);
 596         adev->doorbell.ptr = NULL;
 597 }
 598
 599
 600
 601 /*
 602  * amdgpu_device_wb_*()
 603  * Writeback is the method by which the GPU updates special pages in memory
 604  * with the status of certain GPU events (fences, ring pointers,etc.).
 605  */
 606
 607 /**
 608  * amdgpu_device_wb_fini - Disable Writeback and free memory
 609  *
 610  * @adev: amdgpu_device pointer
 611  *
 612  * Disables Writeback and frees the Writeback memory (all asics).
 613  * Used at driver shutdown.
 614  */
 615 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
 616 {
 617         if (adev->wb.wb_obj) {
 618                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
 619                                       &adev->wb.gpu_addr,
 620                                       (void **)&adev->wb.wb);
 621                 adev->wb.wb_obj = NULL;
 622         }
 623 }
 624
 625 /**
 626  * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
 627  *
 628  * @adev: amdgpu_device pointer
 629  *
 630  * Initializes writeback and allocates writeback memory (all asics).
 631  * Used at driver startup.
 632  * Returns 0 on success or an -error on failure.
 633  */
 634 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
 635 {
 636         int r;
 637
 638         if (adev->wb.wb_obj == NULL) {
 639                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
 640                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
 641                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 642                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
 643                                             (void **)&adev->wb.wb);
 644                 if (r) {
 645                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
 646                         return r;
 647                 }
 648
 649                 adev->wb.num_wb = AMDGPU_MAX_WB;
 650                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
 651
 652                 /* clear wb memory */
 653                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
 654         }
 655
 656         return 0;
 657 }
 658
 659 /**
 660  * amdgpu_device_wb_get - Allocate a wb entry
 661  *
 662  * @adev: amdgpu_device pointer
 663  * @wb: wb index
 664  *
 665  * Allocate a wb slot for use by the driver (all asics).
 666  * Returns 0 on success or -EINVAL on failure.
 667  */
 668 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
 669 {
 670         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 671
 672         if (offset < adev->wb.num_wb) {
 673                 __set_bit(offset, adev->wb.used);
 674                 *wb = offset << 3; /* convert to dw offset */
 675                 return 0;
 676         } else {
 677                 return -EINVAL;
 678         }
 679 }
 680
 681 /**
 682  * amdgpu_device_wb_free - Free a wb entry
 683  *
 684  * @adev: amdgpu_device pointer
 685  * @wb: wb index
 686  *
 687  * Free a wb slot allocated for use by the driver (all asics)
 688  */
 689 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
 690 {
 691         wb >>= 3;
 692         if (wb < adev->wb.num_wb)
 693                 __clear_bit(wb, adev->wb.used);
 694 }
 695
 696 /**
 697  * amdgpu_device_resize_fb_bar - try to resize FB BAR
 698  *
 699  * @adev: amdgpu_device pointer
 700  *
 701  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
 702  * to fail, but if any of the BARs is not accessible after the size we abort
 703  * driver loading by returning -ENODEV.
 704  */
 705 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
 706 {
 707         u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
 708         u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
 709         struct pci_bus *root;
 710         struct resource *res;
 711         unsigned i;
 712         u16 cmd;
 713         int r;
 714
 715         /* Bypass for VF */
 716         if (amdgpu_sriov_vf(adev))
 717                 return 0;
 718
 719         /* Check if the root BUS has 64bit memory resources */
 720         root = adev->pdev->bus;
 721         while (root->parent)
 722                 root = root->parent;
 723
 724         pci_bus_for_each_resource(root, res, i) {
 725                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
 726                     res->start > 0x100000000ull)
 727                         break;
 728         }
 729
 730         /* Trying to resize is pointless without a root hub window above 4GB */
 731         if (!res)
 732                 return 0;
 733
 734         /* Disable memory decoding while we change the BAR addresses and size */
 735         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
 736         pci_write_config_word(adev->pdev, PCI_COMMAND,
 737                               cmd & ~PCI_COMMAND_MEMORY);
 738
 739         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
 740         amdgpu_device_doorbell_fini(adev);
 741         if (adev->asic_type >= CHIP_BONAIRE)
 742                 pci_release_resource(adev->pdev, 2);
 743
 744         pci_release_resource(adev->pdev, 0);
 745
 746         r = pci_resize_resource(adev->pdev, 0, rbar_size);
 747         if (r == -ENOSPC)
 748                 DRM_INFO("Not enough PCI address space for a large BAR.");
 749         else if (r && r != -ENOTSUPP)
 750                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
 751
 752         pci_assign_unassigned_bus_resources(adev->pdev->bus);
 753
 754         /* When the doorbell or fb BAR isn't available we have no chance of
 755          * using the device.
 756          */
 757         r = amdgpu_device_doorbell_init(adev);
 758         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
 759                 return -ENODEV;
 760
 761         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
 762
 763         return 0;
 764 }
 765
 766 /*
 767  * GPU helpers function.
 768  */
 769 /**
 770  * amdgpu_device_need_post - check if the hw need post or not
 771  *
 772  * @adev: amdgpu_device pointer
 773  *
 774  * Check if the asic has been initialized (all asics) at driver startup
 775  * or post is needed if  hw reset is performed.
 776  * Returns true if need or false if not.
 777  */
 778 bool amdgpu_device_need_post(struct amdgpu_device *adev)
 779 {
 780         uint32_t reg;
 781
 782         if (amdgpu_sriov_vf(adev))
 783                 return false;
 784
 785         if (amdgpu_passthrough(adev)) {
 786                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
 787                  * some old smc fw still need driver do vPost otherwise gpu hang, while
 788                  * those smc fw version above 22.15 doesn't have this flaw, so we force
 789                  * vpost executed for smc version below 22.15
 790                  */
 791                 if (adev->asic_type == CHIP_FIJI) {
 792                         int err;
 793                         uint32_t fw_ver;
 794                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
 795                         /* force vPost if error occured */
 796                         if (err)
 797                                 return true;
 798
 799                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
 800                         if (fw_ver < 0x00160e00)
 801                                 return true;
 802                 }
 803         }
 804
 805         if (adev->has_hw_reset) {
 806                 adev->has_hw_reset = false;
 807                 return true;
 808         }
 809
 810         /* bios scratch used on CIK+ */
 811         if (adev->asic_type >= CHIP_BONAIRE)
 812                 return amdgpu_atombios_scratch_need_asic_init(adev);
 813
 814         /* check MEM_SIZE for older asics */
 815         reg = amdgpu_asic_get_config_memsize(adev);
 816
 817         if ((reg != 0) && (reg != 0xffffffff))
 818                 return false;
 819
 820         return true;
 821 }
 822
 823 /* if we get transitioned to only one device, take VGA back */
 824 /**
 825  * amdgpu_device_vga_set_decode - enable/disable vga decode
 826  *
 827  * @cookie: amdgpu_device pointer
 828  * @state: enable/disable vga decode
 829  *
 830  * Enable/disable vga decode (all asics).
 831  * Returns VGA resource flags.
 832  */
 833 static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
 834 {
 835         struct amdgpu_device *adev = cookie;
 836         amdgpu_asic_set_vga_state(adev, state);
 837         if (state)
 838                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
 839                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 840         else
 841                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 842 }
 843
 844 /**
 845  * amdgpu_device_check_block_size - validate the vm block size
 846  *
 847  * @adev: amdgpu_device pointer
 848  *
 849  * Validates the vm block size specified via module parameter.
 850  * The vm block size defines number of bits in page table versus page directory,
 851  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
 852  * page table and the remaining bits are in the page directory.
 853  */
 854 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
 855 {
 856         /* defines number of bits in page table versus page directory,
 857          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
 858          * page table and the remaining bits are in the page directory */
 859         if (amdgpu_vm_block_size == -1)
 860                 return;
 861
 862         if (amdgpu_vm_block_size < 9) {
 863                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
 864                          amdgpu_vm_block_size);
 865                 amdgpu_vm_block_size = -1;
 866         }
 867 }
 868
 869 /**
 870  * amdgpu_device_check_vm_size - validate the vm size
 871  *
 872  * @adev: amdgpu_device pointer
 873  *
 874  * Validates the vm size in GB specified via module parameter.
 875  * The VM size is the size of the GPU virtual memory space in GB.
 876  */
 877 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
 878 {
 879         /* no need to check the default value */
 880         if (amdgpu_vm_size == -1)
 881                 return;
 882
 883         if (amdgpu_vm_size < 1) {
 884                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
 885                          amdgpu_vm_size);
 886                 amdgpu_vm_size = -1;
 887         }
 888 }
 889
 890 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
 891 {
 892         struct sysinfo si;
 893         bool is_os_64 = (sizeof(void *) == 8) ? true : false;
 894         uint64_t total_memory;
 895         uint64_t dram_size_seven_GB = 0x1B8000000;
 896         uint64_t dram_size_three_GB = 0xB8000000;
 897
 898         if (amdgpu_smu_memory_pool_size == 0)
 899                 return;
 900
 901         if (!is_os_64) {
 902                 DRM_WARN("Not 64-bit OS, feature not supported\n");
 903                 goto def_value;
 904         }
 905         si_meminfo(&si);
 906         total_memory = (uint64_t)si.totalram * si.mem_unit;
 907
 908         if ((amdgpu_smu_memory_pool_size == 1) ||
 909                 (amdgpu_smu_memory_pool_size == 2)) {
 910                 if (total_memory < dram_size_three_GB)
 911                         goto def_value1;
 912         } else if ((amdgpu_smu_memory_pool_size == 4) ||
 913                 (amdgpu_smu_memory_pool_size == 8)) {
 914                 if (total_memory < dram_size_seven_GB)
 915                         goto def_value1;
 916         } else {
 917                 DRM_WARN("Smu memory pool size not supported\n");
 918                 goto def_value;
 919         }
 920         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
 921
 922         return;
 923
 924 def_value1:
 925         DRM_WARN("No enough system memory\n");
 926 def_value:
 927         adev->pm.smu_prv_buffer_size = 0;
 928 }
 929
 930 /**
 931  * amdgpu_device_check_arguments - validate module params
 932  *
 933  * @adev: amdgpu_device pointer
 934  *
 935  * Validates certain module parameters and updates
 936  * the associated values used by the driver (all asics).
 937  */
 938 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
 939 {
 940         int ret = 0;
 941
 942         if (amdgpu_sched_jobs < 4) {
 943                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
 944                          amdgpu_sched_jobs);
 945                 amdgpu_sched_jobs = 4;
 946         } else if (!is_power_of_2(amdgpu_sched_jobs)){
 947                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
 948                          amdgpu_sched_jobs);
 949                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
 950         }
 951
 952         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
 953                 /* gart size must be greater or equal to 32M */
 954                 dev_warn(adev->dev, "gart size (%d) too small\n",
 955                          amdgpu_gart_size);
 956                 amdgpu_gart_size = -1;
 957         }
 958
 959         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
 960                 /* gtt size must be greater or equal to 32M */
 961                 dev_warn(adev->dev, "gtt size (%d) too small\n",
 962                                  amdgpu_gtt_size);
 963                 amdgpu_gtt_size = -1;
 964         }
 965
 966         /* valid range is between 4 and 9 inclusive */
 967         if (amdgpu_vm_fragment_size != -1 &&
 968             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
 969                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
 970                 amdgpu_vm_fragment_size = -1;
 971         }
 972
 973         amdgpu_device_check_smu_prv_buffer_size(adev);
 974
 975         amdgpu_device_check_vm_size(adev);
 976
 977         amdgpu_device_check_block_size(adev);
 978
 979         ret = amdgpu_device_get_job_timeout_settings(adev);
 980         if (ret) {
 981                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
 982                 return ret;
 983         }
 984
 985         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
 986
 987         return ret;
 988 }
 989
 990 /**
 991  * amdgpu_switcheroo_set_state - set switcheroo state
 992  *
 993  * @pdev: pci dev pointer
 994  * @state: vga_switcheroo state
 995  *
 996  * Callback for the switcheroo driver.  Suspends or resumes the
 997  * the asics before or after it is powered up using ACPI methods.
 998  */
 999 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1000 {
1001         struct drm_device *dev = pci_get_drvdata(pdev);
1002
1003         if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1004                 return;
1005
1006         if (state == VGA_SWITCHEROO_ON) {
1007                 pr_info("amdgpu: switched on\n");
1008                 /* don't suspend or resume card normally */
1009                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1010
1011                 amdgpu_device_resume(dev, true, true);
1012
1013                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1014                 drm_kms_helper_poll_enable(dev);
1015         } else {
1016                 pr_info("amdgpu: switched off\n");
1017                 drm_kms_helper_poll_disable(dev);
1018                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1019                 amdgpu_device_suspend(dev, true, true);
1020                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1021         }
1022 }
1023
1024 /**
1025  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1026  *
1027  * @pdev: pci dev pointer
1028  *
1029  * Callback for the switcheroo driver.  Check of the switcheroo
1030  * state can be changed.
1031  * Returns true if the state can be changed, false if not.
1032  */
1033 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1034 {
1035         struct drm_device *dev = pci_get_drvdata(pdev);
1036
1037         /*
1038         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1039         * locking inversion with the driver load path. And the access here is
1040         * completely racy anyway. So don't bother with locking for now.
1041         */
1042         return dev->open_count == 0;
1043 }
1044
1045 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1046         .set_gpu_state = amdgpu_switcheroo_set_state,
1047         .reprobe = NULL,
1048         .can_switch = amdgpu_switcheroo_can_switch,
1049 };
1050
1051 /**
1052  * amdgpu_device_ip_set_clockgating_state - set the CG state
1053  *
1054  * @dev: amdgpu_device pointer
1055  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1056  * @state: clockgating state (gate or ungate)
1057  *
1058  * Sets the requested clockgating state for all instances of
1059  * the hardware IP specified.
1060  * Returns the error code from the last instance.
1061  */
1062 int amdgpu_device_ip_set_clockgating_state(void *dev,
1063                                            enum amd_ip_block_type block_type,
1064                                            enum amd_clockgating_state state)
1065 {
1066         struct amdgpu_device *adev = dev;
1067         int i, r = 0;
1068
1069         for (i = 0; i < adev->num_ip_blocks; i++) {
1070                 if (!adev->ip_blocks[i].status.valid)
1071                         continue;
1072                 if (adev->ip_blocks[i].version->type != block_type)
1073                         continue;
1074                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1075                         continue;
1076                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1077                         (void *)adev, state);
1078                 if (r)
1079                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1080                                   adev->ip_blocks[i].version->funcs->name, r);
1081         }
1082         return r;
1083 }
1084
1085 /**
1086  * amdgpu_device_ip_set_powergating_state - set the PG state
1087  *
1088  * @dev: amdgpu_device pointer
1089  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1090  * @state: powergating state (gate or ungate)
1091  *
1092  * Sets the requested powergating state for all instances of
1093  * the hardware IP specified.
1094  * Returns the error code from the last instance.
1095  */
1096 int amdgpu_device_ip_set_powergating_state(void *dev,
1097                                            enum amd_ip_block_type block_type,
1098                                            enum amd_powergating_state state)
1099 {
1100         struct amdgpu_device *adev = dev;
1101         int i, r = 0;
1102
1103         for (i = 0; i < adev->num_ip_blocks; i++) {
1104                 if (!adev->ip_blocks[i].status.valid)
1105                         continue;
1106                 if (adev->ip_blocks[i].version->type != block_type)
1107                         continue;
1108                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1109                         continue;
1110                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1111                         (void *)adev, state);
1112                 if (r)
1113                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1114                                   adev->ip_blocks[i].version->funcs->name, r);
1115         }
1116         return r;
1117 }
1118
1119 /**
1120  * amdgpu_device_ip_get_clockgating_state - get the CG state
1121  *
1122  * @adev: amdgpu_device pointer
1123  * @flags: clockgating feature flags
1124  *
1125  * Walks the list of IPs on the device and updates the clockgating
1126  * flags for each IP.
1127  * Updates @flags with the feature flags for each hardware IP where
1128  * clockgating is enabled.
1129  */
1130 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1131                                             u32 *flags)
1132 {
1133         int i;
1134
1135         for (i = 0; i < adev->num_ip_blocks; i++) {
1136                 if (!adev->ip_blocks[i].status.valid)
1137                         continue;
1138                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1139                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1140         }
1141 }
1142
1143 /**
1144  * amdgpu_device_ip_wait_for_idle - wait for idle
1145  *
1146  * @adev: amdgpu_device pointer
1147  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1148  *
1149  * Waits for the request hardware IP to be idle.
1150  * Returns 0 for success or a negative error code on failure.
1151  */
1152 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1153                                    enum amd_ip_block_type block_type)
1154 {
1155         int i, r;
1156
1157         for (i = 0; i < adev->num_ip_blocks; i++) {
1158                 if (!adev->ip_blocks[i].status.valid)
1159                         continue;
1160                 if (adev->ip_blocks[i].version->type == block_type) {
1161                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1162                         if (r)
1163                                 return r;
1164                         break;
1165                 }
1166         }
1167         return 0;
1168
1169 }
1170
1171 /**
1172  * amdgpu_device_ip_is_idle - is the hardware IP idle
1173  *
1174  * @adev: amdgpu_device pointer
1175  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1176  *
1177  * Check if the hardware IP is idle or not.
1178  * Returns true if it the IP is idle, false if not.
1179  */
1180 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1181                               enum amd_ip_block_type block_type)
1182 {
1183         int i;
1184
1185         for (i = 0; i < adev->num_ip_blocks; i++) {
1186                 if (!adev->ip_blocks[i].status.valid)
1187                         continue;
1188                 if (adev->ip_blocks[i].version->type == block_type)
1189                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1190         }
1191         return true;
1192
1193 }
1194
1195 /**
1196  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1197  *
1198  * @adev: amdgpu_device pointer
1199  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1200  *
1201  * Returns a pointer to the hardware IP block structure
1202  * if it exists for the asic, otherwise NULL.
1203  */
1204 struct amdgpu_ip_block *
1205 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1206                               enum amd_ip_block_type type)
1207 {
1208         int i;
1209
1210         for (i = 0; i < adev->num_ip_blocks; i++)
1211                 if (adev->ip_blocks[i].version->type == type)
1212                         return &adev->ip_blocks[i];
1213
1214         return NULL;
1215 }
1216
1217 /**
1218  * amdgpu_device_ip_block_version_cmp
1219  *
1220  * @adev: amdgpu_device pointer
1221  * @type: enum amd_ip_block_type
1222  * @major: major version
1223  * @minor: minor version
1224  *
1225  * return 0 if equal or greater
1226  * return 1 if smaller or the ip_block doesn't exist
1227  */
1228 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1229                                        enum amd_ip_block_type type,
1230                                        u32 major, u32 minor)
1231 {
1232         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1233
1234         if (ip_block && ((ip_block->version->major > major) ||
1235                         ((ip_block->version->major == major) &&
1236                         (ip_block->version->minor >= minor))))
1237                 return 0;
1238
1239         return 1;
1240 }
1241
1242 /**
1243  * amdgpu_device_ip_block_add
1244  *
1245  * @adev: amdgpu_device pointer
1246  * @ip_block_version: pointer to the IP to add
1247  *
1248  * Adds the IP block driver information to the collection of IPs
1249  * on the asic.
1250  */
1251 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1252                                const struct amdgpu_ip_block_version *ip_block_version)
1253 {
1254         if (!ip_block_version)
1255                 return -EINVAL;
1256
1257         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1258                   ip_block_version->funcs->name);
1259
1260         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1261
1262         return 0;
1263 }
1264
1265 /**
1266  * amdgpu_device_enable_virtual_display - enable virtual display feature
1267  *
1268  * @adev: amdgpu_device pointer
1269  *
1270  * Enabled the virtual display feature if the user has enabled it via
1271  * the module parameter virtual_display.  This feature provides a virtual
1272  * display hardware on headless boards or in virtualized environments.
1273  * This function parses and validates the configuration string specified by
1274  * the user and configues the virtual display configuration (number of
1275  * virtual connectors, crtcs, etc.) specified.
1276  */
1277 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1278 {
1279         adev->enable_virtual_display = false;
1280
1281         if (amdgpu_virtual_display) {
1282                 struct drm_device *ddev = adev->ddev;
1283                 const char *pci_address_name = pci_name(ddev->pdev);
1284                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1285
1286                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1287                 pciaddstr_tmp = pciaddstr;
1288                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1289                         pciaddname = strsep(&pciaddname_tmp, ",");
1290                         if (!strcmp("all", pciaddname)
1291                             || !strcmp(pci_address_name, pciaddname)) {
1292                                 long num_crtc;
1293                                 int res = -1;
1294
1295                                 adev->enable_virtual_display = true;
1296
1297                                 if (pciaddname_tmp)
1298                                         res = kstrtol(pciaddname_tmp, 10,
1299                                                       &num_crtc);
1300
1301                                 if (!res) {
1302                                         if (num_crtc < 1)
1303                                                 num_crtc = 1;
1304                                         if (num_crtc > 6)
1305                                                 num_crtc = 6;
1306                                         adev->mode_info.num_crtc = num_crtc;
1307                                 } else {
1308                                         adev->mode_info.num_crtc = 1;
1309                                 }
1310                                 break;
1311                         }
1312                 }
1313
1314                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1315                          amdgpu_virtual_display, pci_address_name,
1316                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1317
1318                 kfree(pciaddstr);
1319         }
1320 }
1321
1322 /**
1323  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1324  *
1325  * @adev: amdgpu_device pointer
1326  *
1327  * Parses the asic configuration parameters specified in the gpu info
1328  * firmware and makes them availale to the driver for use in configuring
1329  * the asic.
1330  * Returns 0 on success, -EINVAL on failure.
1331  */
1332 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1333 {
1334         const char *chip_name;
1335         char fw_name[30];
1336         int err;
1337         const struct gpu_info_firmware_header_v1_0 *hdr;
1338
1339         adev->firmware.gpu_info_fw = NULL;
1340
1341         switch (adev->asic_type) {
1342         case CHIP_TOPAZ:
1343         case CHIP_TONGA:
1344         case CHIP_FIJI:
1345         case CHIP_POLARIS10:
1346         case CHIP_POLARIS11:
1347         case CHIP_POLARIS12:
1348         case CHIP_VEGAM:
1349         case CHIP_CARRIZO:
1350         case CHIP_STONEY:
1351 #ifdef CONFIG_DRM_AMDGPU_SI
1352         case CHIP_VERDE:
1353         case CHIP_TAHITI:
1354         case CHIP_PITCAIRN:
1355         case CHIP_OLAND:
1356         case CHIP_HAINAN:
1357 #endif
1358 #ifdef CONFIG_DRM_AMDGPU_CIK
1359         case CHIP_BONAIRE:
1360         case CHIP_HAWAII:
1361         case CHIP_KAVERI:
1362         case CHIP_KABINI:
1363         case CHIP_MULLINS:
1364 #endif
1365         case CHIP_VEGA20:
1366         default:
1367                 return 0;
1368         case CHIP_VEGA10:
1369                 chip_name = "vega10";
1370                 break;
1371         case CHIP_VEGA12:
1372                 chip_name = "vega12";
1373                 break;
1374         case CHIP_RAVEN:
1375                 if (adev->rev_id >= 8)
1376                         chip_name = "raven2";
1377                 else if (adev->pdev->device == 0x15d8)
1378                         chip_name = "picasso";
1379                 else
1380                         chip_name = "raven";
1381                 break;
1382         case CHIP_NAVI10:
1383                 chip_name = "navi10";
1384                 break;
1385         }
1386
1387         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1388         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1389         if (err) {
1390                 dev_err(adev->dev,
1391                         "Failed to load gpu_info firmware \"%s\"\n",
1392                         fw_name);
1393                 goto out;
1394         }
1395         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1396         if (err) {
1397                 dev_err(adev->dev,
1398                         "Failed to validate gpu_info firmware \"%s\"\n",
1399                         fw_name);
1400                 goto out;
1401         }
1402
1403         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1404         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1405
1406         switch (hdr->version_major) {
1407         case 1:
1408         {
1409                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1410                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1411                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1412
1413                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1414                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1415                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1416                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1417                 adev->gfx.config.max_texture_channel_caches =
1418                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
1419                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1420                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1421                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1422                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1423                 adev->gfx.config.double_offchip_lds_buf =
1424                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1425                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1426                 adev->gfx.cu_info.max_waves_per_simd =
1427                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1428                 adev->gfx.cu_info.max_scratch_slots_per_cu =
1429                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1430                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1431                 if (hdr->version_minor == 1) {
1432                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1433                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1434                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1435                         adev->gfx.config.num_sc_per_sh =
1436                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
1437                         adev->gfx.config.num_packer_per_sc =
1438                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
1439                 }
1440                 break;
1441         }
1442         default:
1443                 dev_err(adev->dev,
1444                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1445                 err = -EINVAL;
1446                 goto out;
1447         }
1448 out:
1449         return err;
1450 }
1451
1452 /**
1453  * amdgpu_device_ip_early_init - run early init for hardware IPs
1454  *
1455  * @adev: amdgpu_device pointer
1456  *
1457  * Early initialization pass for hardware IPs.  The hardware IPs that make
1458  * up each asic are discovered each IP's early_init callback is run.  This
1459  * is the first stage in initializing the asic.
1460  * Returns 0 on success, negative error code on failure.
1461  */
1462 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1463 {
1464         int i, r;
1465
1466         amdgpu_device_enable_virtual_display(adev);
1467
1468         switch (adev->asic_type) {
1469         case CHIP_TOPAZ:
1470         case CHIP_TONGA:
1471         case CHIP_FIJI:
1472         case CHIP_POLARIS10:
1473         case CHIP_POLARIS11:
1474         case CHIP_POLARIS12:
1475         case CHIP_VEGAM:
1476         case CHIP_CARRIZO:
1477         case CHIP_STONEY:
1478                 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1479                         adev->family = AMDGPU_FAMILY_CZ;
1480                 else
1481                         adev->family = AMDGPU_FAMILY_VI;
1482
1483                 r = vi_set_ip_blocks(adev);
1484                 if (r)
1485                         return r;
1486                 break;
1487 #ifdef CONFIG_DRM_AMDGPU_SI
1488         case CHIP_VERDE:
1489         case CHIP_TAHITI:
1490         case CHIP_PITCAIRN:
1491         case CHIP_OLAND:
1492         case CHIP_HAINAN:
1493                 adev->family = AMDGPU_FAMILY_SI;
1494                 r = si_set_ip_blocks(adev);
1495                 if (r)
1496                         return r;
1497                 break;
1498 #endif
1499 #ifdef CONFIG_DRM_AMDGPU_CIK
1500         case CHIP_BONAIRE:
1501         case CHIP_HAWAII:
1502         case CHIP_KAVERI:
1503         case CHIP_KABINI:
1504         case CHIP_MULLINS:
1505                 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1506                         adev->family = AMDGPU_FAMILY_CI;
1507                 else
1508                         adev->family = AMDGPU_FAMILY_KV;
1509
1510                 r = cik_set_ip_blocks(adev);
1511                 if (r)
1512                         return r;
1513                 break;
1514 #endif
1515         case CHIP_VEGA10:
1516         case CHIP_VEGA12:
1517         case CHIP_VEGA20:
1518         case CHIP_RAVEN:
1519                 if (adev->asic_type == CHIP_RAVEN)
1520                         adev->family = AMDGPU_FAMILY_RV;
1521                 else
1522                         adev->family = AMDGPU_FAMILY_AI;
1523
1524                 r = soc15_set_ip_blocks(adev);
1525                 if (r)
1526                         return r;
1527                 break;
1528         default:
1529                 /* FIXME: not supported yet */
1530                 return -EINVAL;
1531         }
1532
1533         r = amdgpu_device_parse_gpu_info_fw(adev);
1534         if (r)
1535                 return r;
1536
1537         amdgpu_amdkfd_device_probe(adev);
1538
1539         if (amdgpu_sriov_vf(adev)) {
1540                 r = amdgpu_virt_request_full_gpu(adev, true);
1541                 if (r)
1542                         return -EAGAIN;
1543
1544                 /* query the reg access mode at the very beginning */
1545                 amdgpu_virt_init_reg_access_mode(adev);
1546         }
1547
1548         adev->pm.pp_feature = amdgpu_pp_feature_mask;
1549         if (amdgpu_sriov_vf(adev))
1550                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1551
1552         for (i = 0; i < adev->num_ip_blocks; i++) {
1553                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1554                         DRM_ERROR("disabled ip block: %d <%s>\n",
1555                                   i, adev->ip_blocks[i].version->funcs->name);
1556                         adev->ip_blocks[i].status.valid = false;
1557                 } else {
1558                         if (adev->ip_blocks[i].version->funcs->early_init) {
1559                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1560                                 if (r == -ENOENT) {
1561                                         adev->ip_blocks[i].status.valid = false;
1562                                 } else if (r) {
1563                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
1564                                                   adev->ip_blocks[i].version->funcs->name, r);
1565                                         return r;
1566                                 } else {
1567                                         adev->ip_blocks[i].status.valid = true;
1568                                 }
1569                         } else {
1570                                 adev->ip_blocks[i].status.valid = true;
1571                         }
1572                 }
1573                 /* get the vbios after the asic_funcs are set up */
1574                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
1575                         /* Read BIOS */
1576                         if (!amdgpu_get_bios(adev))
1577                                 return -EINVAL;
1578
1579                         r = amdgpu_atombios_init(adev);
1580                         if (r) {
1581                                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
1582                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
1583                                 return r;
1584                         }
1585                 }
1586         }
1587
1588         adev->cg_flags &= amdgpu_cg_mask;
1589         adev->pg_flags &= amdgpu_pg_mask;
1590
1591         return 0;
1592 }
1593
1594 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
1595 {
1596         int i, r;
1597
1598         for (i = 0; i < adev->num_ip_blocks; i++) {
1599                 if (!adev->ip_blocks[i].status.sw)
1600                         continue;
1601                 if (adev->ip_blocks[i].status.hw)
1602                         continue;
1603                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1604                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
1605                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
1606                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1607                         if (r) {
1608                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1609                                           adev->ip_blocks[i].version->funcs->name, r);
1610                                 return r;
1611                         }
1612                         adev->ip_blocks[i].status.hw = true;
1613                 }
1614         }
1615
1616         return 0;
1617 }
1618
1619 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
1620 {
1621         int i, r;
1622
1623         for (i = 0; i < adev->num_ip_blocks; i++) {
1624                 if (!adev->ip_blocks[i].status.sw)
1625                         continue;
1626                 if (adev->ip_blocks[i].status.hw)
1627                         continue;
1628                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1629                 if (r) {
1630                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1631                                   adev->ip_blocks[i].version->funcs->name, r);
1632                         return r;
1633                 }
1634                 adev->ip_blocks[i].status.hw = true;
1635         }
1636
1637         return 0;
1638 }
1639
1640 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
1641 {
1642         int r = 0;
1643         int i;
1644         uint32_t smu_version;
1645
1646         if (adev->asic_type >= CHIP_VEGA10) {
1647                 for (i = 0; i < adev->num_ip_blocks; i++) {
1648                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
1649                                 if (adev->in_gpu_reset || adev->in_suspend) {
1650                                         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset)
1651                                                 break; /* sriov gpu reset, psp need to do hw_init before IH because of hw limit */
1652                                         r = adev->ip_blocks[i].version->funcs->resume(adev);
1653                                         if (r) {
1654                                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
1655                                                           adev->ip_blocks[i].version->funcs->name, r);
1656                                                 return r;
1657                                         }
1658                                 } else {
1659                                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1660                                         if (r) {
1661                                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1662                                                   adev->ip_blocks[i].version->funcs->name, r);
1663                                                 return r;
1664                                         }
1665                                 }
1666                                 adev->ip_blocks[i].status.hw = true;
1667                         }
1668                 }
1669         }
1670         r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
1671
1672         return r;
1673 }
1674
1675 /**
1676  * amdgpu_device_ip_init - run init for hardware IPs
1677  *
1678  * @adev: amdgpu_device pointer
1679  *
1680  * Main initialization pass for hardware IPs.  The list of all the hardware
1681  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
1682  * are run.  sw_init initializes the software state associated with each IP
1683  * and hw_init initializes the hardware associated with each IP.
1684  * Returns 0 on success, negative error code on failure.
1685  */
1686 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
1687 {
1688         int i, r;
1689
1690         r = amdgpu_ras_init(adev);
1691         if (r)
1692                 return r;
1693
1694         for (i = 0; i < adev->num_ip_blocks; i++) {
1695                 if (!adev->ip_blocks[i].status.valid)
1696                         continue;
1697                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1698                 if (r) {
1699                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1700                                   adev->ip_blocks[i].version->funcs->name, r);
1701                         goto init_failed;
1702                 }
1703                 adev->ip_blocks[i].status.sw = true;
1704
1705                 /* need to do gmc hw init early so we can allocate gpu mem */
1706                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1707                         r = amdgpu_device_vram_scratch_init(adev);
1708                         if (r) {
1709                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1710                                 goto init_failed;
1711                         }
1712                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1713                         if (r) {
1714                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
1715                                 goto init_failed;
1716                         }
1717                         r = amdgpu_device_wb_init(adev);
1718                         if (r) {
1719                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
1720                                 goto init_failed;
1721                         }
1722                         adev->ip_blocks[i].status.hw = true;
1723
1724                         /* right after GMC hw init, we create CSA */
1725                         if (amdgpu_sriov_vf(adev)) {
1726                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
1727                                                                 AMDGPU_GEM_DOMAIN_VRAM,
1728                                                                 AMDGPU_CSA_SIZE);
1729                                 if (r) {
1730                                         DRM_ERROR("allocate CSA failed %d\n", r);
1731                                         goto init_failed;
1732                                 }
1733                         }
1734                 }
1735         }
1736
1737         r = amdgpu_ib_pool_init(adev);
1738         if (r) {
1739                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
1740                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
1741                 goto init_failed;
1742         }
1743
1744         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
1745         if (r)
1746                 goto init_failed;
1747
1748         r = amdgpu_device_ip_hw_init_phase1(adev);
1749         if (r)
1750                 goto init_failed;
1751
1752         r = amdgpu_device_fw_loading(adev);
1753         if (r)
1754                 goto init_failed;
1755
1756         r = amdgpu_device_ip_hw_init_phase2(adev);
1757         if (r)
1758                 goto init_failed;
1759
1760         if (adev->gmc.xgmi.num_physical_nodes > 1)
1761                 amdgpu_xgmi_add_device(adev);
1762         amdgpu_amdkfd_device_init(adev);
1763
1764 init_failed:
1765         if (amdgpu_sriov_vf(adev)) {
1766                 if (!r)
1767                         amdgpu_virt_init_data_exchange(adev);
1768                 amdgpu_virt_release_full_gpu(adev, true);
1769         }
1770
1771         return r;
1772 }
1773
1774 /**
1775  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
1776  *
1777  * @adev: amdgpu_device pointer
1778  *
1779  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
1780  * this function before a GPU reset.  If the value is retained after a
1781  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
1782  */
1783 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
1784 {
1785         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1786 }
1787
1788 /**
1789  * amdgpu_device_check_vram_lost - check if vram is valid
1790  *
1791  * @adev: amdgpu_device pointer
1792  *
1793  * Checks the reset magic value written to the gart pointer in VRAM.
1794  * The driver calls this after a GPU reset to see if the contents of
1795  * VRAM is lost or now.
1796  * returns true if vram is lost, false if not.
1797  */
1798 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
1799 {
1800         return !!memcmp(adev->gart.ptr, adev->reset_magic,
1801                         AMDGPU_RESET_MAGIC_NUM);
1802 }
1803
1804 /**
1805  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
1806  *
1807  * @adev: amdgpu_device pointer
1808  *
1809  * The list of all the hardware IPs that make up the asic is walked and the
1810  * set_clockgating_state callbacks are run.
1811  * Late initialization pass enabling clockgating for hardware IPs.
1812  * Fini or suspend, pass disabling clockgating for hardware IPs.
1813  * Returns 0 on success, negative error code on failure.
1814  */
1815
1816 static int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
1817                                                 enum amd_clockgating_state state)
1818 {
1819         int i, j, r;
1820
1821         if (amdgpu_emu_mode == 1)
1822                 return 0;
1823
1824         for (j = 0; j < adev->num_ip_blocks; j++) {
1825                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
1826                 if (!adev->ip_blocks[i].status.late_initialized)
1827                         continue;
1828                 /* skip CG for VCE/UVD, it's handled specially */
1829                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1830                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1831                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
1832                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1833                         /* enable clockgating to save power */
1834                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1835                                                                                      state);
1836                         if (r) {
1837                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1838                                           adev->ip_blocks[i].version->funcs->name, r);
1839                                 return r;
1840                         }
1841                 }
1842         }
1843
1844         return 0;
1845 }
1846
1847 static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_powergating_state state)
1848 {
1849         int i, j, r;
1850
1851         if (amdgpu_emu_mode == 1)
1852                 return 0;
1853
1854         for (j = 0; j < adev->num_ip_blocks; j++) {
1855                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
1856                 if (!adev->ip_blocks[i].status.late_initialized)
1857                         continue;
1858                 /* skip CG for VCE/UVD, it's handled specially */
1859                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1860                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1861                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
1862                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
1863                         /* enable powergating to save power */
1864                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
1865                                                                                         state);
1866                         if (r) {
1867                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
1868                                           adev->ip_blocks[i].version->funcs->name, r);
1869                                 return r;
1870                         }
1871                 }
1872         }
1873         return 0;
1874 }
1875
1876 static int amdgpu_device_enable_mgpu_fan_boost(void)
1877 {
1878         struct amdgpu_gpu_instance *gpu_ins;
1879         struct amdgpu_device *adev;
1880         int i, ret = 0;
1881
1882         mutex_lock(&mgpu_info.mutex);
1883
1884         /*
1885          * MGPU fan boost feature should be enabled
1886          * only when there are two or more dGPUs in
1887          * the system
1888          */
1889         if (mgpu_info.num_dgpu < 2)
1890                 goto out;
1891
1892         for (i = 0; i < mgpu_info.num_dgpu; i++) {
1893                 gpu_ins = &(mgpu_info.gpu_ins[i]);
1894                 adev = gpu_ins->adev;
1895                 if (!(adev->flags & AMD_IS_APU) &&
1896                     !gpu_ins->mgpu_fan_enabled &&
1897                     adev->powerplay.pp_funcs &&
1898                     adev->powerplay.pp_funcs->enable_mgpu_fan_boost) {
1899                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
1900                         if (ret)
1901                                 break;
1902
1903                         gpu_ins->mgpu_fan_enabled = 1;
1904                 }
1905         }
1906
1907 out:
1908         mutex_unlock(&mgpu_info.mutex);
1909
1910         return ret;
1911 }
1912
1913 /**
1914  * amdgpu_device_ip_late_init - run late init for hardware IPs
1915  *
1916  * @adev: amdgpu_device pointer
1917  *
1918  * Late initialization pass for hardware IPs.  The list of all the hardware
1919  * IPs that make up the asic is walked and the late_init callbacks are run.
1920  * late_init covers any special initialization that an IP requires
1921  * after all of the have been initialized or something that needs to happen
1922  * late in the init process.
1923  * Returns 0 on success, negative error code on failure.
1924  */
1925 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
1926 {
1927         int i = 0, r;
1928
1929         for (i = 0; i < adev->num_ip_blocks; i++) {
1930                 if (!adev->ip_blocks[i].status.hw)
1931                         continue;
1932                 if (adev->ip_blocks[i].version->funcs->late_init) {
1933                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1934                         if (r) {
1935                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1936                                           adev->ip_blocks[i].version->funcs->name, r);
1937                                 return r;
1938                         }
1939                 }
1940                 adev->ip_blocks[i].status.late_initialized = true;
1941         }
1942
1943         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
1944         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
1945
1946         amdgpu_device_fill_reset_magic(adev);
1947
1948         r = amdgpu_device_enable_mgpu_fan_boost();
1949         if (r)
1950                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
1951
1952         /* set to low pstate by default */
1953         amdgpu_xgmi_set_pstate(adev, 0);
1954
1955         return 0;
1956 }
1957
1958 /**
1959  * amdgpu_device_ip_fini - run fini for hardware IPs
1960  *
1961  * @adev: amdgpu_device pointer
1962  *
1963  * Main teardown pass for hardware IPs.  The list of all the hardware
1964  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
1965  * are run.  hw_fini tears down the hardware associated with each IP
1966  * and sw_fini tears down any software state associated with each IP.
1967  * Returns 0 on success, negative error code on failure.
1968  */
1969 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
1970 {
1971         int i, r;
1972
1973         amdgpu_ras_pre_fini(adev);
1974
1975         if (adev->gmc.xgmi.num_physical_nodes > 1)
1976                 amdgpu_xgmi_remove_device(adev);
1977
1978         amdgpu_amdkfd_device_fini(adev);
1979
1980         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
1981         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
1982
1983         /* need to disable SMC first */
1984         for (i = 0; i < adev->num_ip_blocks; i++) {
1985                 if (!adev->ip_blocks[i].status.hw)
1986                         continue;
1987                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
1988                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1989                         /* XXX handle errors */
1990                         if (r) {
1991                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1992                                           adev->ip_blocks[i].version->funcs->name, r);
1993                         }
1994                         adev->ip_blocks[i].status.hw = false;
1995                         break;
1996                 }
1997         }
1998
1999         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2000                 if (!adev->ip_blocks[i].status.hw)
2001                         continue;
2002
2003                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2004                 /* XXX handle errors */
2005                 if (r) {
2006                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2007                                   adev->ip_blocks[i].version->funcs->name, r);
2008                 }
2009
2010                 adev->ip_blocks[i].status.hw = false;
2011         }
2012
2013
2014         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2015                 if (!adev->ip_blocks[i].status.sw)
2016                         continue;
2017
2018                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2019                         amdgpu_ucode_free_bo(adev);
2020                         amdgpu_free_static_csa(&adev->virt.csa_obj);
2021                         amdgpu_device_wb_fini(adev);
2022                         amdgpu_device_vram_scratch_fini(adev);
2023                         amdgpu_ib_pool_fini(adev);
2024                 }
2025
2026                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2027                 /* XXX handle errors */
2028                 if (r) {
2029                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2030                                   adev->ip_blocks[i].version->funcs->name, r);
2031                 }
2032                 adev->ip_blocks[i].status.sw = false;
2033                 adev->ip_blocks[i].status.valid = false;
2034         }
2035
2036         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2037                 if (!adev->ip_blocks[i].status.late_initialized)
2038                         continue;
2039                 if (adev->ip_blocks[i].version->funcs->late_fini)
2040                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2041                 adev->ip_blocks[i].status.late_initialized = false;
2042         }
2043
2044         amdgpu_ras_fini(adev);
2045
2046         if (amdgpu_sriov_vf(adev))
2047                 if (amdgpu_virt_release_full_gpu(adev, false))
2048                         DRM_ERROR("failed to release exclusive mode on fini\n");
2049
2050         return 0;
2051 }
2052
2053 /**
2054  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2055  *
2056  * @work: work_struct.
2057  */
2058 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2059 {
2060         struct amdgpu_device *adev =
2061                 container_of(work, struct amdgpu_device, delayed_init_work.work);
2062         int r;
2063
2064         r = amdgpu_ib_ring_tests(adev);
2065         if (r)
2066                 DRM_ERROR("ib ring test failed (%d).\n", r);
2067 }
2068
2069 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2070 {
2071         struct amdgpu_device *adev =
2072                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2073
2074         mutex_lock(&adev->gfx.gfx_off_mutex);
2075         if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) {
2076                 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2077                         adev->gfx.gfx_off_state = true;
2078         }
2079         mutex_unlock(&adev->gfx.gfx_off_mutex);
2080 }
2081
2082 /**
2083  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2084  *
2085  * @adev: amdgpu_device pointer
2086  *
2087  * Main suspend function for hardware IPs.  The list of all the hardware
2088  * IPs that make up the asic is walked, clockgating is disabled and the
2089  * suspend callbacks are run.  suspend puts the hardware and software state
2090  * in each IP into a state suitable for suspend.
2091  * Returns 0 on success, negative error code on failure.
2092  */
2093 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2094 {
2095         int i, r;
2096
2097         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2098         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2099
2100         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2101                 if (!adev->ip_blocks[i].status.valid)
2102                         continue;
2103                 /* displays are handled separately */
2104                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
2105                         /* XXX handle errors */
2106                         r = adev->ip_blocks[i].version->funcs->suspend(adev);
2107                         /* XXX handle errors */
2108                         if (r) {
2109                                 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2110                                           adev->ip_blocks[i].version->funcs->name, r);
2111                         }
2112                 }
2113         }
2114
2115         return 0;
2116 }
2117
2118 /**
2119  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2120  *
2121  * @adev: amdgpu_device pointer
2122  *
2123  * Main suspend function for hardware IPs.  The list of all the hardware
2124  * IPs that make up the asic is walked, clockgating is disabled and the
2125  * suspend callbacks are run.  suspend puts the hardware and software state
2126  * in each IP into a state suitable for suspend.
2127  * Returns 0 on success, negative error code on failure.
2128  */
2129 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2130 {
2131         int i, r;
2132
2133         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2134                 if (!adev->ip_blocks[i].status.valid)
2135                         continue;
2136                 /* displays are handled in phase1 */
2137                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2138                         continue;
2139                 /* XXX handle errors */
2140                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2141                 /* XXX handle errors */
2142                 if (r) {
2143                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2144                                   adev->ip_blocks[i].version->funcs->name, r);
2145                 }
2146         }
2147
2148         return 0;
2149 }
2150
2151 /**
2152  * amdgpu_device_ip_suspend - run suspend for hardware IPs
2153  *
2154  * @adev: amdgpu_device pointer
2155  *
2156  * Main suspend function for hardware IPs.  The list of all the hardware
2157  * IPs that make up the asic is walked, clockgating is disabled and the
2158  * suspend callbacks are run.  suspend puts the hardware and software state
2159  * in each IP into a state suitable for suspend.
2160  * Returns 0 on success, negative error code on failure.
2161  */
2162 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2163 {
2164         int r;
2165
2166         if (amdgpu_sriov_vf(adev))
2167                 amdgpu_virt_request_full_gpu(adev, false);
2168
2169         r = amdgpu_device_ip_suspend_phase1(adev);
2170         if (r)
2171                 return r;
2172         r = amdgpu_device_ip_suspend_phase2(adev);
2173
2174         if (amdgpu_sriov_vf(adev))
2175                 amdgpu_virt_release_full_gpu(adev, false);
2176
2177         return r;
2178 }
2179
2180 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2181 {
2182         int i, r;
2183
2184         static enum amd_ip_block_type ip_order[] = {
2185                 AMD_IP_BLOCK_TYPE_GMC,
2186                 AMD_IP_BLOCK_TYPE_COMMON,
2187                 AMD_IP_BLOCK_TYPE_PSP,
2188                 AMD_IP_BLOCK_TYPE_IH,
2189         };
2190
2191         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2192                 int j;
2193                 struct amdgpu_ip_block *block;
2194
2195                 for (j = 0; j < adev->num_ip_blocks; j++) {
2196                         block = &adev->ip_blocks[j];
2197
2198                         if (block->version->type != ip_order[i] ||
2199                                 !block->status.valid)
2200                                 continue;
2201
2202                         r = block->version->funcs->hw_init(adev);
2203                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2204                         if (r)
2205                                 return r;
2206                 }
2207         }
2208
2209         return 0;
2210 }
2211
2212 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2213 {
2214         int i, r;
2215
2216         static enum amd_ip_block_type ip_order[] = {
2217                 AMD_IP_BLOCK_TYPE_SMC,
2218                 AMD_IP_BLOCK_TYPE_DCE,
2219                 AMD_IP_BLOCK_TYPE_GFX,
2220                 AMD_IP_BLOCK_TYPE_SDMA,
2221                 AMD_IP_BLOCK_TYPE_UVD,
2222                 AMD_IP_BLOCK_TYPE_VCE
2223         };
2224
2225         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2226                 int j;
2227                 struct amdgpu_ip_block *block;
2228
2229                 for (j = 0; j < adev->num_ip_blocks; j++) {
2230                         block = &adev->ip_blocks[j];
2231
2232                         if (block->version->type != ip_order[i] ||
2233                                 !block->status.valid)
2234                                 continue;
2235
2236                         r = block->version->funcs->hw_init(adev);
2237                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2238                         if (r)
2239                                 return r;
2240                 }
2241         }
2242
2243         return 0;
2244 }
2245
2246 /**
2247  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
2248  *
2249  * @adev: amdgpu_device pointer
2250  *
2251  * First resume function for hardware IPs.  The list of all the hardware
2252  * IPs that make up the asic is walked and the resume callbacks are run for
2253  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
2254  * after a suspend and updates the software state as necessary.  This
2255  * function is also used for restoring the GPU after a GPU reset.
2256  * Returns 0 on success, negative error code on failure.
2257  */
2258 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
2259 {
2260         int i, r;
2261
2262         for (i = 0; i < adev->num_ip_blocks; i++) {
2263                 if (!adev->ip_blocks[i].status.valid)
2264                         continue;
2265                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2266                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2267                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2268                         r = adev->ip_blocks[i].version->funcs->resume(adev);
2269                         if (r) {
2270                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
2271                                           adev->ip_blocks[i].version->funcs->name, r);
2272                                 return r;
2273                         }
2274                 }
2275         }
2276
2277         return 0;
2278 }
2279
2280 /**
2281  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
2282  *
2283  * @adev: amdgpu_device pointer
2284  *
2285  * First resume function for hardware IPs.  The list of all the hardware
2286  * IPs that make up the asic is walked and the resume callbacks are run for
2287  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
2288  * functional state after a suspend and updates the software state as
2289  * necessary.  This function is also used for restoring the GPU after a GPU
2290  * reset.
2291  * Returns 0 on success, negative error code on failure.
2292  */
2293 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
2294 {
2295         int i, r;
2296
2297         for (i = 0; i < adev->num_ip_blocks; i++) {
2298                 if (!adev->ip_blocks[i].status.valid)
2299                         continue;
2300                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2301                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2302                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
2303                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
2304                         continue;
2305                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2306                 if (r) {
2307                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2308                                   adev->ip_blocks[i].version->funcs->name, r);
2309                         return r;
2310                 }
2311         }
2312
2313         return 0;
2314 }
2315
2316 /**
2317  * amdgpu_device_ip_resume - run resume for hardware IPs
2318  *
2319  * @adev: amdgpu_device pointer
2320  *
2321  * Main resume function for hardware IPs.  The hardware IPs
2322  * are split into two resume functions because they are
2323  * are also used in in recovering from a GPU reset and some additional
2324  * steps need to be take between them.  In this case (S3/S4) they are
2325  * run sequentially.
2326  * Returns 0 on success, negative error code on failure.
2327  */
2328 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
2329 {
2330         int r;
2331
2332         r = amdgpu_device_ip_resume_phase1(adev);
2333         if (r)
2334                 return r;
2335
2336         r = amdgpu_device_fw_loading(adev);
2337         if (r)
2338                 return r;
2339
2340         r = amdgpu_device_ip_resume_phase2(adev);
2341
2342         return r;
2343 }
2344
2345 /**
2346  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
2347  *
2348  * @adev: amdgpu_device pointer
2349  *
2350  * Query the VBIOS data tables to determine if the board supports SR-IOV.
2351  */
2352 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2353 {
2354         if (amdgpu_sriov_vf(adev)) {
2355                 if (adev->is_atom_fw) {
2356                         if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2357                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2358                 } else {
2359                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2360                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2361                 }
2362
2363                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
2364                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2365         }
2366 }
2367
2368 /**
2369  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
2370  *
2371  * @asic_type: AMD asic type
2372  *
2373  * Check if there is DC (new modesetting infrastructre) support for an asic.
2374  * returns true if DC has support, false if not.
2375  */
2376 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
2377 {
2378         switch (asic_type) {
2379 #if defined(CONFIG_DRM_AMD_DC)
2380         case CHIP_BONAIRE:
2381         case CHIP_KAVERI:
2382         case CHIP_KABINI:
2383         case CHIP_MULLINS:
2384                 /*
2385                  * We have systems in the wild with these ASICs that require
2386                  * LVDS and VGA support which is not supported with DC.
2387                  *
2388                  * Fallback to the non-DC driver here by default so as not to
2389                  * cause regressions.
2390                  */
2391                 return amdgpu_dc > 0;
2392         case CHIP_HAWAII:
2393         case CHIP_CARRIZO:
2394         case CHIP_STONEY:
2395         case CHIP_POLARIS10:
2396         case CHIP_POLARIS11:
2397         case CHIP_POLARIS12:
2398         case CHIP_VEGAM:
2399         case CHIP_TONGA:
2400         case CHIP_FIJI:
2401         case CHIP_VEGA10:
2402         case CHIP_VEGA12:
2403         case CHIP_VEGA20:
2404 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
2405         case CHIP_RAVEN:
2406 #endif
2407                 return amdgpu_dc != 0;
2408 #endif
2409         default:
2410                 return false;
2411         }
2412 }
2413
2414 /**
2415  * amdgpu_device_has_dc_support - check if dc is supported
2416  *
2417  * @adev: amdgpu_device_pointer
2418  *
2419  * Returns true for supported, false for not supported
2420  */
2421 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
2422 {
2423         if (amdgpu_sriov_vf(adev))
2424                 return false;
2425
2426         return amdgpu_device_asic_has_dc_support(adev->asic_type);
2427 }
2428
2429
2430 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
2431 {
2432         struct amdgpu_device *adev =
2433                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
2434
2435         adev->asic_reset_res =  amdgpu_asic_reset(adev);
2436         if (adev->asic_reset_res)
2437                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
2438                          adev->asic_reset_res, adev->ddev->unique);
2439 }
2440
2441
2442 /**
2443  * amdgpu_device_init - initialize the driver
2444  *
2445  * @adev: amdgpu_device pointer
2446  * @ddev: drm dev pointer
2447  * @pdev: pci dev pointer
2448  * @flags: driver flags
2449  *
2450  * Initializes the driver info and hw (all asics).
2451  * Returns 0 for success or an error on failure.
2452  * Called at driver startup.
2453  */
2454 int amdgpu_device_init(struct amdgpu_device *adev,
2455                        struct drm_device *ddev,
2456                        struct pci_dev *pdev,
2457                        uint32_t flags)
2458 {
2459         int r, i;
2460         bool runtime = false;
2461         u32 max_MBps;
2462
2463         adev->shutdown = false;
2464         adev->dev = &pdev->dev;
2465         adev->ddev = ddev;
2466         adev->pdev = pdev;
2467         adev->flags = flags;
2468         adev->asic_type = flags & AMD_ASIC_MASK;
2469         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2470         if (amdgpu_emu_mode == 1)
2471                 adev->usec_timeout *= 2;
2472         adev->gmc.gart_size = 512 * 1024 * 1024;
2473         adev->accel_working = false;
2474         adev->num_rings = 0;
2475         adev->mman.buffer_funcs = NULL;
2476         adev->mman.buffer_funcs_ring = NULL;
2477         adev->vm_manager.vm_pte_funcs = NULL;
2478         adev->vm_manager.vm_pte_num_rqs = 0;
2479         adev->gmc.gmc_funcs = NULL;
2480         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2481         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2482
2483         adev->smc_rreg = &amdgpu_invalid_rreg;
2484         adev->smc_wreg = &amdgpu_invalid_wreg;
2485         adev->pcie_rreg = &amdgpu_invalid_rreg;
2486         adev->pcie_wreg = &amdgpu_invalid_wreg;
2487         adev->pciep_rreg = &amdgpu_invalid_rreg;
2488         adev->pciep_wreg = &amdgpu_invalid_wreg;
2489         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2490         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2491         adev->didt_rreg = &amdgpu_invalid_rreg;
2492         adev->didt_wreg = &amdgpu_invalid_wreg;
2493         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2494         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
2495         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2496         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2497
2498         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2499                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2500                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
2501
2502         /* mutex initialization are all done here so we
2503          * can recall function without having locking issues */
2504         atomic_set(&adev->irq.ih.lock, 0);
2505         mutex_init(&adev->firmware.mutex);
2506         mutex_init(&adev->pm.mutex);
2507         mutex_init(&adev->gfx.gpu_clock_mutex);
2508         mutex_init(&adev->srbm_mutex);
2509         mutex_init(&adev->gfx.pipe_reserve_mutex);
2510         mutex_init(&adev->gfx.gfx_off_mutex);
2511         mutex_init(&adev->grbm_idx_mutex);
2512         mutex_init(&adev->mn_lock);
2513         mutex_init(&adev->virt.vf_errors.lock);
2514         hash_init(adev->mn_hash);
2515         mutex_init(&adev->lock_reset);
2516         mutex_init(&adev->virt.dpm_mutex);
2517
2518         r = amdgpu_device_check_arguments(adev);
2519         if (r)
2520                 return r;
2521
2522         spin_lock_init(&adev->mmio_idx_lock);
2523         spin_lock_init(&adev->smc_idx_lock);
2524         spin_lock_init(&adev->pcie_idx_lock);
2525         spin_lock_init(&adev->uvd_ctx_idx_lock);
2526         spin_lock_init(&adev->didt_idx_lock);
2527         spin_lock_init(&adev->gc_cac_idx_lock);
2528         spin_lock_init(&adev->se_cac_idx_lock);
2529         spin_lock_init(&adev->audio_endpt_idx_lock);
2530         spin_lock_init(&adev->mm_stats.lock);
2531
2532         INIT_LIST_HEAD(&adev->shadow_list);
2533         mutex_init(&adev->shadow_list_lock);
2534
2535         INIT_LIST_HEAD(&adev->ring_lru_list);
2536         spin_lock_init(&adev->ring_lru_list_lock);
2537
2538         INIT_DELAYED_WORK(&adev->delayed_init_work,
2539                           amdgpu_device_delayed_init_work_handler);
2540         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
2541                           amdgpu_device_delay_enable_gfx_off);
2542
2543         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
2544
2545         adev->gfx.gfx_off_req_count = 1;
2546         adev->pm.ac_power = power_supply_is_system_supplied() > 0 ? true : false;
2547
2548         /* Registers mapping */
2549         /* TODO: block userspace mapping of io register */
2550         if (adev->asic_type >= CHIP_BONAIRE) {
2551                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2552                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2553         } else {
2554                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2555                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2556         }
2557
2558         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2559         if (adev->rmmio == NULL) {
2560                 return -ENOMEM;
2561         }
2562         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2563         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2564
2565         /* io port mapping */
2566         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2567                 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2568                         adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2569                         adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2570                         break;
2571                 }
2572         }
2573         if (adev->rio_mem == NULL)
2574                 DRM_INFO("PCI I/O BAR is not found.\n");
2575
2576         amdgpu_device_get_pcie_info(adev);
2577
2578         /* early init functions */
2579         r = amdgpu_device_ip_early_init(adev);
2580         if (r)
2581                 return r;
2582
2583         /* doorbell bar mapping and doorbell index init*/
2584         amdgpu_device_doorbell_init(adev);
2585
2586         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2587         /* this will fail for cards that aren't VGA class devices, just
2588          * ignore it */
2589         vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
2590
2591         if (amdgpu_device_is_px(ddev))
2592                 runtime = true;
2593         if (!pci_is_thunderbolt_attached(adev->pdev))
2594                 vga_switcheroo_register_client(adev->pdev,
2595                                                &amdgpu_switcheroo_ops, runtime);
2596         if (runtime)
2597                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2598
2599         if (amdgpu_emu_mode == 1) {
2600                 /* post the asic on emulation mode */
2601                 emu_soc_asic_init(adev);
2602                 goto fence_driver_init;
2603         }
2604
2605         /* detect if we are with an SRIOV vbios */
2606         amdgpu_device_detect_sriov_bios(adev);
2607
2608         /* check if we need to reset the asic
2609          *  E.g., driver was not cleanly unloaded previously, etc.
2610          */
2611         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
2612                 r = amdgpu_asic_reset(adev);
2613                 if (r) {
2614                         dev_err(adev->dev, "asic reset on init failed\n");
2615                         goto failed;
2616                 }
2617         }
2618
2619         /* Post card if necessary */
2620         if (amdgpu_device_need_post(adev)) {
2621                 if (!adev->bios) {
2622                         dev_err(adev->dev, "no vBIOS found\n");
2623                         r = -EINVAL;
2624                         goto failed;
2625                 }
2626                 DRM_INFO("GPU posting now...\n");
2627                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2628                 if (r) {
2629                         dev_err(adev->dev, "gpu post error!\n");
2630                         goto failed;
2631                 }
2632         }
2633
2634         if (adev->is_atom_fw) {
2635                 /* Initialize clocks */
2636                 r = amdgpu_atomfirmware_get_clock_info(adev);
2637                 if (r) {
2638                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
2639                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2640                         goto failed;
2641                 }
2642         } else {
2643                 /* Initialize clocks */
2644                 r = amdgpu_atombios_get_clock_info(adev);
2645                 if (r) {
2646                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2647                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2648                         goto failed;
2649                 }
2650                 /* init i2c buses */
2651                 if (!amdgpu_device_has_dc_support(adev))
2652                         amdgpu_atombios_i2c_init(adev);
2653         }
2654
2655 fence_driver_init:
2656         /* Fence driver */
2657         r = amdgpu_fence_driver_init(adev);
2658         if (r) {
2659                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2660                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2661                 goto failed;
2662         }
2663
2664         /* init the mode config */
2665         drm_mode_config_init(adev->ddev);
2666
2667         r = amdgpu_device_ip_init(adev);
2668         if (r) {
2669                 /* failed in exclusive mode due to timeout */
2670                 if (amdgpu_sriov_vf(adev) &&
2671                     !amdgpu_sriov_runtime(adev) &&
2672                     amdgpu_virt_mmio_blocked(adev) &&
2673                     !amdgpu_virt_wait_reset(adev)) {
2674                         dev_err(adev->dev, "VF exclusive mode timeout\n");
2675                         /* Don't send request since VF is inactive. */
2676                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
2677                         adev->virt.ops = NULL;
2678                         r = -EAGAIN;
2679                         goto failed;
2680                 }
2681                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
2682                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2683                 if (amdgpu_virt_request_full_gpu(adev, false))
2684                         amdgpu_virt_release_full_gpu(adev, false);
2685                 goto failed;
2686         }
2687
2688         adev->accel_working = true;
2689
2690         amdgpu_vm_check_compute_bug(adev);
2691
2692         /* Initialize the buffer migration limit. */
2693         if (amdgpu_moverate >= 0)
2694                 max_MBps = amdgpu_moverate;
2695         else
2696                 max_MBps = 8; /* Allow 8 MB/s. */
2697         /* Get a log2 for easy divisions. */
2698         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2699
2700         amdgpu_fbdev_init(adev);
2701
2702         if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))
2703                 amdgpu_pm_virt_sysfs_init(adev);
2704
2705         r = amdgpu_pm_sysfs_init(adev);
2706         if (r)
2707                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2708
2709         r = amdgpu_ucode_sysfs_init(adev);
2710         if (r)
2711                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
2712
2713         r = amdgpu_debugfs_gem_init(adev);
2714         if (r)
2715                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2716
2717         r = amdgpu_debugfs_regs_init(adev);
2718         if (r)
2719                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2720
2721         r = amdgpu_debugfs_firmware_init(adev);
2722         if (r)
2723                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2724
2725         r = amdgpu_debugfs_init(adev);
2726         if (r)
2727                 DRM_ERROR("Creating debugfs files failed (%d).\n", r);
2728
2729         if ((amdgpu_testing & 1)) {
2730                 if (adev->accel_working)
2731                         amdgpu_test_moves(adev);
2732                 else
2733                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2734         }
2735         if (amdgpu_benchmarking) {
2736                 if (adev->accel_working)
2737                         amdgpu_benchmark(adev, amdgpu_benchmarking);
2738                 else
2739                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2740         }
2741
2742         /* enable clockgating, etc. after ib tests, etc. since some blocks require
2743          * explicit gating rather than handling it automatically.
2744          */
2745         r = amdgpu_device_ip_late_init(adev);
2746         if (r) {
2747                 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
2748                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2749                 goto failed;
2750         }
2751
2752         /* must succeed. */
2753         amdgpu_ras_resume(adev);
2754
2755         queue_delayed_work(system_wq, &adev->delayed_init_work,
2756                            msecs_to_jiffies(AMDGPU_RESUME_MS));
2757
2758         r = device_create_file(adev->dev, &dev_attr_pcie_replay_count);
2759         if (r) {
2760                 dev_err(adev->dev, "Could not create pcie_replay_count");
2761                 return r;
2762         }
2763
2764         r = amdgpu_pmu_init(adev);
2765         if (r)
2766                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
2767
2768         return 0;
2769
2770 failed:
2771         amdgpu_vf_error_trans_all(adev);
2772         if (runtime)
2773                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2774
2775         return r;
2776 }
2777
2778 /**
2779  * amdgpu_device_fini - tear down the driver
2780  *
2781  * @adev: amdgpu_device pointer
2782  *
2783  * Tear down the driver info (all asics).
2784  * Called at driver shutdown.
2785  */
2786 void amdgpu_device_fini(struct amdgpu_device *adev)
2787 {
2788         int r;
2789
2790         DRM_INFO("amdgpu: finishing device.\n");
2791         adev->shutdown = true;
2792         /* disable all interrupts */
2793         amdgpu_irq_disable_all(adev);
2794         if (adev->mode_info.mode_config_initialized){
2795                 if (!amdgpu_device_has_dc_support(adev))
2796                         drm_helper_force_disable_all(adev->ddev);
2797                 else
2798                         drm_atomic_helper_shutdown(adev->ddev);
2799         }
2800         amdgpu_fence_driver_fini(adev);
2801         amdgpu_pm_sysfs_fini(adev);
2802         amdgpu_fbdev_fini(adev);
2803         r = amdgpu_device_ip_fini(adev);
2804         if (adev->firmware.gpu_info_fw) {
2805                 release_firmware(adev->firmware.gpu_info_fw);
2806                 adev->firmware.gpu_info_fw = NULL;
2807         }
2808         adev->accel_working = false;
2809         cancel_delayed_work_sync(&adev->delayed_init_work);
2810         /* free i2c buses */
2811         if (!amdgpu_device_has_dc_support(adev))
2812                 amdgpu_i2c_fini(adev);
2813
2814         if (amdgpu_emu_mode != 1)
2815                 amdgpu_atombios_fini(adev);
2816
2817         kfree(adev->bios);
2818         adev->bios = NULL;
2819         if (!pci_is_thunderbolt_attached(adev->pdev))
2820                 vga_switcheroo_unregister_client(adev->pdev);
2821         if (adev->flags & AMD_IS_PX)
2822                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2823         vga_client_register(adev->pdev, NULL, NULL, NULL);
2824         if (adev->rio_mem)
2825                 pci_iounmap(adev->pdev, adev->rio_mem);
2826         adev->rio_mem = NULL;
2827         iounmap(adev->rmmio);
2828         adev->rmmio = NULL;
2829         amdgpu_device_doorbell_fini(adev);
2830         if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))
2831                 amdgpu_pm_virt_sysfs_fini(adev);
2832
2833         amdgpu_debugfs_regs_cleanup(adev);
2834         device_remove_file(adev->dev, &dev_attr_pcie_replay_count);
2835         amdgpu_ucode_sysfs_fini(adev);
2836         amdgpu_pmu_fini(adev);
2837 }
2838
2839
2840 /*
2841  * Suspend & resume.
2842  */
2843 /**
2844  * amdgpu_device_suspend - initiate device suspend
2845  *
2846  * @dev: drm dev pointer
2847  * @suspend: suspend state
2848  * @fbcon : notify the fbdev of suspend
2849  *
2850  * Puts the hw in the suspend state (all asics).
2851  * Returns 0 for success or an error on failure.
2852  * Called at driver suspend.
2853  */
2854 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2855 {
2856         struct amdgpu_device *adev;
2857         struct drm_crtc *crtc;
2858         struct drm_connector *connector;
2859         int r;
2860
2861         if (dev == NULL || dev->dev_private == NULL) {
2862                 return -ENODEV;
2863         }
2864
2865         adev = dev->dev_private;
2866
2867         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2868                 return 0;
2869
2870         adev->in_suspend = true;
2871         drm_kms_helper_poll_disable(dev);
2872
2873         if (fbcon)
2874                 amdgpu_fbdev_set_suspend(adev, 1);
2875
2876         cancel_delayed_work_sync(&adev->delayed_init_work);
2877
2878         if (!amdgpu_device_has_dc_support(adev)) {
2879                 /* turn off display hw */
2880                 drm_modeset_lock_all(dev);
2881                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2882                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2883                 }
2884                 drm_modeset_unlock_all(dev);
2885                         /* unpin the front buffers and cursors */
2886                 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2887                         struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2888                         struct drm_framebuffer *fb = crtc->primary->fb;
2889                         struct amdgpu_bo *robj;
2890
2891                         if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
2892                                 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2893                                 r = amdgpu_bo_reserve(aobj, true);
2894                                 if (r == 0) {
2895                                         amdgpu_bo_unpin(aobj);
2896                                         amdgpu_bo_unreserve(aobj);
2897                                 }
2898                         }
2899
2900                         if (fb == NULL || fb->obj[0] == NULL) {
2901                                 continue;
2902                         }
2903                         robj = gem_to_amdgpu_bo(fb->obj[0]);
2904                         /* don't unpin kernel fb objects */
2905                         if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2906                                 r = amdgpu_bo_reserve(robj, true);
2907                                 if (r == 0) {
2908                                         amdgpu_bo_unpin(robj);
2909                                         amdgpu_bo_unreserve(robj);
2910                                 }
2911                         }
2912                 }
2913         }
2914
2915         amdgpu_amdkfd_suspend(adev);
2916
2917         amdgpu_ras_suspend(adev);
2918
2919         r = amdgpu_device_ip_suspend_phase1(adev);
2920
2921         /* evict vram memory */
2922         amdgpu_bo_evict_vram(adev);
2923
2924         amdgpu_fence_driver_suspend(adev);
2925
2926         r = amdgpu_device_ip_suspend_phase2(adev);
2927
2928         /* evict remaining vram memory
2929          * This second call to evict vram is to evict the gart page table
2930          * using the CPU.
2931          */
2932         amdgpu_bo_evict_vram(adev);
2933
2934         pci_save_state(dev->pdev);
2935         if (suspend) {
2936                 /* Shut down the device */
2937                 pci_disable_device(dev->pdev);
2938                 pci_set_power_state(dev->pdev, PCI_D3hot);
2939         } else {
2940                 r = amdgpu_asic_reset(adev);
2941                 if (r)
2942                         DRM_ERROR("amdgpu asic reset failed\n");
2943         }
2944
2945         return 0;
2946 }
2947
2948 /**
2949  * amdgpu_device_resume - initiate device resume
2950  *
2951  * @dev: drm dev pointer
2952  * @resume: resume state
2953  * @fbcon : notify the fbdev of resume
2954  *
2955  * Bring the hw back to operating state (all asics).
2956  * Returns 0 for success or an error on failure.
2957  * Called at driver resume.
2958  */
2959 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2960 {
2961         struct drm_connector *connector;
2962         struct amdgpu_device *adev = dev->dev_private;
2963         struct drm_crtc *crtc;
2964         int r = 0;
2965
2966         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2967                 return 0;
2968
2969         if (resume) {
2970                 pci_set_power_state(dev->pdev, PCI_D0);
2971                 pci_restore_state(dev->pdev);
2972                 r = pci_enable_device(dev->pdev);
2973                 if (r)
2974                         return r;
2975         }
2976
2977         /* post card */
2978         if (amdgpu_device_need_post(adev)) {
2979                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2980                 if (r)
2981                         DRM_ERROR("amdgpu asic init failed\n");
2982         }
2983
2984         r = amdgpu_device_ip_resume(adev);
2985         if (r) {
2986                 DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
2987                 return r;
2988         }
2989         amdgpu_fence_driver_resume(adev);
2990
2991
2992         r = amdgpu_device_ip_late_init(adev);
2993         if (r)
2994                 return r;
2995
2996         queue_delayed_work(system_wq, &adev->delayed_init_work,
2997                            msecs_to_jiffies(AMDGPU_RESUME_MS));
2998
2999         if (!amdgpu_device_has_dc_support(adev)) {
3000                 /* pin cursors */
3001                 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3002                         struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
3003
3004                         if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3005                                 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
3006                                 r = amdgpu_bo_reserve(aobj, true);
3007                                 if (r == 0) {
3008                                         r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
3009                                         if (r != 0)
3010                                                 DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
3011                                         amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
3012                                         amdgpu_bo_unreserve(aobj);
3013                                 }
3014                         }
3015                 }
3016         }
3017         r = amdgpu_amdkfd_resume(adev);
3018         if (r)
3019                 return r;
3020
3021         /* Make sure IB tests flushed */
3022         flush_delayed_work(&adev->delayed_init_work);
3023
3024         /* blat the mode back in */
3025         if (fbcon) {
3026                 if (!amdgpu_device_has_dc_support(adev)) {
3027                         /* pre DCE11 */
3028                         drm_helper_resume_force_mode(dev);
3029
3030                         /* turn on display hw */
3031                         drm_modeset_lock_all(dev);
3032                         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
3033                                 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
3034                         }
3035                         drm_modeset_unlock_all(dev);
3036                 }
3037                 amdgpu_fbdev_set_suspend(adev, 0);
3038         }
3039
3040         drm_kms_helper_poll_enable(dev);
3041
3042         amdgpu_ras_resume(adev);
3043
3044         /*
3045          * Most of the connector probing functions try to acquire runtime pm
3046          * refs to ensure that the GPU is powered on when connector polling is
3047          * performed. Since we're calling this from a runtime PM callback,
3048          * trying to acquire rpm refs will cause us to deadlock.
3049          *
3050          * Since we're guaranteed to be holding the rpm lock, it's safe to
3051          * temporarily disable the rpm helpers so this doesn't deadlock us.
3052          */
3053 #ifdef CONFIG_PM
3054         dev->dev->power.disable_depth++;
3055 #endif
3056         if (!amdgpu_device_has_dc_support(adev))
3057                 drm_helper_hpd_irq_event(dev);
3058         else
3059                 drm_kms_helper_hotplug_event(dev);
3060 #ifdef CONFIG_PM
3061         dev->dev->power.disable_depth--;
3062 #endif
3063         adev->in_suspend = false;
3064
3065         return 0;
3066 }
3067
3068 /**
3069  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
3070  *
3071  * @adev: amdgpu_device pointer
3072  *
3073  * The list of all the hardware IPs that make up the asic is walked and
3074  * the check_soft_reset callbacks are run.  check_soft_reset determines
3075  * if the asic is still hung or not.
3076  * Returns true if any of the IPs are still in a hung state, false if not.
3077  */
3078 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
3079 {
3080         int i;
3081         bool asic_hang = false;
3082
3083         if (amdgpu_sriov_vf(adev))
3084                 return true;
3085
3086         if (amdgpu_asic_need_full_reset(adev))
3087                 return true;
3088
3089         for (i = 0; i < adev->num_ip_blocks; i++) {
3090                 if (!adev->ip_blocks[i].status.valid)
3091                         continue;
3092                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
3093                         adev->ip_blocks[i].status.hang =
3094                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
3095                 if (adev->ip_blocks[i].status.hang) {
3096                         DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
3097                         asic_hang = true;
3098                 }
3099         }
3100         return asic_hang;
3101 }
3102
3103 /**
3104  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
3105  *
3106  * @adev: amdgpu_device pointer
3107  *
3108  * The list of all the hardware IPs that make up the asic is walked and the
3109  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
3110  * handles any IP specific hardware or software state changes that are
3111  * necessary for a soft reset to succeed.
3112  * Returns 0 on success, negative error code on failure.
3113  */
3114 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
3115 {
3116         int i, r = 0;
3117
3118         for (i = 0; i < adev->num_ip_blocks; i++) {
3119                 if (!adev->ip_blocks[i].status.valid)
3120                         continue;
3121                 if (adev->ip_blocks[i].status.hang &&
3122                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
3123                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
3124                         if (r)
3125                                 return r;
3126                 }
3127         }
3128
3129         return 0;
3130 }
3131
3132 /**
3133  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
3134  *
3135  * @adev: amdgpu_device pointer
3136  *
3137  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
3138  * reset is necessary to recover.
3139  * Returns true if a full asic reset is required, false if not.
3140  */
3141 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
3142 {
3143         int i;
3144
3145         if (amdgpu_asic_need_full_reset(adev))
3146                 return true;
3147
3148         for (i = 0; i < adev->num_ip_blocks; i++) {
3149                 if (!adev->ip_blocks[i].status.valid)
3150                         continue;
3151                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
3152                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
3153                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
3154                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
3155                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3156                         if (adev->ip_blocks[i].status.hang) {
3157                                 DRM_INFO("Some block need full reset!\n");
3158                                 return true;
3159                         }
3160                 }
3161         }
3162         return false;
3163 }
3164
3165 /**
3166  * amdgpu_device_ip_soft_reset - do a soft reset
3167  *
3168  * @adev: amdgpu_device pointer
3169  *
3170  * The list of all the hardware IPs that make up the asic is walked and the
3171  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
3172  * IP specific hardware or software state changes that are necessary to soft
3173  * reset the IP.
3174  * Returns 0 on success, negative error code on failure.
3175  */
3176 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
3177 {
3178         int i, r = 0;
3179
3180         for (i = 0; i < adev->num_ip_blocks; i++) {
3181                 if (!adev->ip_blocks[i].status.valid)
3182                         continue;
3183                 if (adev->ip_blocks[i].status.hang &&
3184                     adev->ip_blocks[i].version->funcs->soft_reset) {
3185                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
3186                         if (r)
3187                                 return r;
3188                 }
3189         }
3190
3191         return 0;
3192 }
3193
3194 /**
3195  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
3196  *
3197  * @adev: amdgpu_device pointer
3198  *
3199  * The list of all the hardware IPs that make up the asic is walked and the
3200  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
3201  * handles any IP specific hardware or software state changes that are
3202  * necessary after the IP has been soft reset.
3203  * Returns 0 on success, negative error code on failure.
3204  */
3205 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
3206 {
3207         int i, r = 0;
3208
3209         for (i = 0; i < adev->num_ip_blocks; i++) {
3210                 if (!adev->ip_blocks[i].status.valid)
3211                         continue;
3212                 if (adev->ip_blocks[i].status.hang &&
3213                     adev->ip_blocks[i].version->funcs->post_soft_reset)
3214                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
3215                 if (r)
3216                         return r;
3217         }
3218
3219         return 0;
3220 }
3221
3222 /**
3223  * amdgpu_device_recover_vram - Recover some VRAM contents
3224  *
3225  * @adev: amdgpu_device pointer
3226  *
3227  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
3228  * restore things like GPUVM page tables after a GPU reset where
3229  * the contents of VRAM might be lost.
3230  *
3231  * Returns:
3232  * 0 on success, negative error code on failure.
3233  */
3234 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
3235 {
3236         struct dma_fence *fence = NULL, *next = NULL;
3237         struct amdgpu_bo *shadow;
3238         long r = 1, tmo;
3239
3240         if (amdgpu_sriov_runtime(adev))
3241                 tmo = msecs_to_jiffies(8000);
3242         else
3243                 tmo = msecs_to_jiffies(100);
3244
3245         DRM_INFO("recover vram bo from shadow start\n");
3246         mutex_lock(&adev->shadow_list_lock);
3247         list_for_each_entry(shadow, &adev->shadow_list, shadow_list) {
3248
3249                 /* No need to recover an evicted BO */
3250                 if (shadow->tbo.mem.mem_type != TTM_PL_TT ||
3251                     shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET ||
3252                     shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM)
3253                         continue;
3254
3255                 r = amdgpu_bo_restore_shadow(shadow, &next);
3256                 if (r)
3257                         break;
3258
3259                 if (fence) {
3260                         tmo = dma_fence_wait_timeout(fence, false, tmo);
3261                         dma_fence_put(fence);
3262                         fence = next;
3263                         if (tmo == 0) {
3264                                 r = -ETIMEDOUT;
3265                                 break;
3266                         } else if (tmo < 0) {
3267                                 r = tmo;
3268                                 break;
3269                         }
3270                 } else {
3271                         fence = next;
3272                 }
3273         }
3274         mutex_unlock(&adev->shadow_list_lock);
3275
3276         if (fence)
3277                 tmo = dma_fence_wait_timeout(fence, false, tmo);
3278         dma_fence_put(fence);
3279
3280         if (r < 0 || tmo <= 0) {
3281                 DRM_ERROR("recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
3282                 return -EIO;
3283         }
3284
3285         DRM_INFO("recover vram bo from shadow done\n");
3286         return 0;
3287 }
3288
3289
3290 /**
3291  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
3292  *
3293  * @adev: amdgpu device pointer
3294  * @from_hypervisor: request from hypervisor
3295  *
3296  * do VF FLR and reinitialize Asic
3297  * return 0 means succeeded otherwise failed
3298  */
3299 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
3300                                      bool from_hypervisor)
3301 {
3302         int r;
3303
3304         if (from_hypervisor)
3305                 r = amdgpu_virt_request_full_gpu(adev, true);
3306         else
3307                 r = amdgpu_virt_reset_gpu(adev);
3308         if (r)
3309                 return r;
3310
3311         amdgpu_amdkfd_pre_reset(adev);
3312
3313         /* Resume IP prior to SMC */
3314         r = amdgpu_device_ip_reinit_early_sriov(adev);
3315         if (r)
3316                 goto error;
3317
3318         /* we need recover gart prior to run SMC/CP/SDMA resume */
3319         amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
3320
3321         r = amdgpu_device_fw_loading(adev);
3322         if (r)
3323                 return r;
3324
3325         /* now we are okay to resume SMC/CP/SDMA */
3326         r = amdgpu_device_ip_reinit_late_sriov(adev);
3327         if (r)
3328                 goto error;
3329
3330         amdgpu_irq_gpu_reset_resume_helper(adev);
3331         r = amdgpu_ib_ring_tests(adev);
3332         amdgpu_amdkfd_post_reset(adev);
3333
3334 error:
3335         amdgpu_virt_init_data_exchange(adev);
3336         amdgpu_virt_release_full_gpu(adev, true);
3337         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
3338                 atomic_inc(&adev->vram_lost_counter);
3339                 r = amdgpu_device_recover_vram(adev);
3340         }
3341
3342         return r;
3343 }
3344
3345 /**
3346  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
3347  *
3348  * @adev: amdgpu device pointer
3349  *
3350  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
3351  * a hung GPU.
3352  */
3353 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
3354 {
3355         if (!amdgpu_device_ip_check_soft_reset(adev)) {
3356                 DRM_INFO("Timeout, but no hardware hang detected.\n");
3357                 return false;
3358         }
3359
3360         if (amdgpu_gpu_recovery == 0)
3361                 goto disabled;
3362
3363         if (amdgpu_sriov_vf(adev))
3364                 return true;
3365
3366         if (amdgpu_gpu_recovery == -1) {
3367                 switch (adev->asic_type) {
3368                 case CHIP_BONAIRE:
3369                 case CHIP_HAWAII:
3370                 case CHIP_TOPAZ:
3371                 case CHIP_TONGA:
3372                 case CHIP_FIJI:
3373                 case CHIP_POLARIS10:
3374                 case CHIP_POLARIS11:
3375                 case CHIP_POLARIS12:
3376                 case CHIP_VEGAM:
3377                 case CHIP_VEGA20:
3378                 case CHIP_VEGA10:
3379                 case CHIP_VEGA12:
3380                         break;
3381                 default:
3382                         goto disabled;
3383                 }
3384         }
3385
3386         return true;
3387
3388 disabled:
3389                 DRM_INFO("GPU recovery disabled.\n");
3390                 return false;
3391 }
3392
3393
3394 static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
3395                                         struct amdgpu_job *job,
3396                                         bool *need_full_reset_arg)
3397 {
3398         int i, r = 0;
3399         bool need_full_reset  = *need_full_reset_arg;
3400
3401         /* block all schedulers and reset given job's ring */
3402         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3403                 struct amdgpu_ring *ring = adev->rings[i];
3404
3405                 if (!ring || !ring->sched.thread)
3406                         continue;
3407
3408                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
3409                 amdgpu_fence_driver_force_completion(ring);
3410         }
3411
3412         if(job)
3413                 drm_sched_increase_karma(&job->base);
3414
3415         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
3416         if (!amdgpu_sriov_vf(adev)) {
3417
3418                 if (!need_full_reset)
3419                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
3420
3421                 if (!need_full_reset) {
3422                         amdgpu_device_ip_pre_soft_reset(adev);
3423                         r = amdgpu_device_ip_soft_reset(adev);
3424                         amdgpu_device_ip_post_soft_reset(adev);
3425                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
3426                                 DRM_INFO("soft reset failed, will fallback to full reset!\n");
3427                                 need_full_reset = true;
3428                         }
3429                 }
3430
3431                 if (need_full_reset)
3432                         r = amdgpu_device_ip_suspend(adev);
3433
3434                 *need_full_reset_arg = need_full_reset;
3435         }
3436
3437         return r;
3438 }
3439
3440 static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
3441                                struct list_head *device_list_handle,
3442                                bool *need_full_reset_arg)
3443 {
3444         struct amdgpu_device *tmp_adev = NULL;
3445         bool need_full_reset = *need_full_reset_arg, vram_lost = false;
3446         int r = 0;
3447
3448         /*
3449          * ASIC reset has to be done on all HGMI hive nodes ASAP
3450          * to allow proper links negotiation in FW (within 1 sec)
3451          */
3452         if (need_full_reset) {
3453                 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3454                         /* For XGMI run all resets in parallel to speed up the process */
3455                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
3456                                 if (!queue_work(system_highpri_wq, &tmp_adev->xgmi_reset_work))
3457                                         r = -EALREADY;
3458                         } else
3459                                 r = amdgpu_asic_reset(tmp_adev);
3460
3461                         if (r) {
3462                                 DRM_ERROR("ASIC reset failed with error, %d for drm dev, %s",
3463                                          r, tmp_adev->ddev->unique);
3464                                 break;
3465                         }
3466                 }
3467
3468                 /* For XGMI wait for all PSP resets to complete before proceed */
3469                 if (!r) {
3470                         list_for_each_entry(tmp_adev, device_list_handle,
3471                                             gmc.xgmi.head) {
3472                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
3473                                         flush_work(&tmp_adev->xgmi_reset_work);
3474                                         r = tmp_adev->asic_reset_res;
3475                                         if (r)
3476                                                 break;
3477                                 }
3478                         }
3479
3480                         list_for_each_entry(tmp_adev, device_list_handle,
3481                                         gmc.xgmi.head) {
3482                                 amdgpu_ras_reserve_bad_pages(tmp_adev);
3483                         }
3484                 }
3485         }
3486
3487
3488         list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3489                 if (need_full_reset) {
3490                         /* post card */
3491                         if (amdgpu_atom_asic_init(tmp_adev->mode_info.atom_context))
3492                                 DRM_WARN("asic atom init failed!");
3493
3494                         if (!r) {
3495                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
3496                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
3497                                 if (r)
3498                                         goto out;
3499
3500                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
3501                                 if (vram_lost) {
3502                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
3503                                         atomic_inc(&tmp_adev->vram_lost_counter);
3504                                 }
3505
3506                                 r = amdgpu_gtt_mgr_recover(
3507                                         &tmp_adev->mman.bdev.man[TTM_PL_TT]);
3508                                 if (r)
3509                                         goto out;
3510
3511                                 r = amdgpu_device_fw_loading(tmp_adev);
3512                                 if (r)
3513                                         return r;
3514
3515                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
3516                                 if (r)
3517                                         goto out;
3518
3519                                 if (vram_lost)
3520                                         amdgpu_device_fill_reset_magic(tmp_adev);
3521
3522                                 r = amdgpu_device_ip_late_init(tmp_adev);
3523                                 if (r)
3524                                         goto out;
3525
3526                                 /* must succeed. */
3527                                 amdgpu_ras_resume(tmp_adev);
3528
3529                                 /* Update PSP FW topology after reset */
3530                                 if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1)
3531                                         r = amdgpu_xgmi_update_topology(hive, tmp_adev);
3532                         }
3533                 }
3534
3535
3536 out:
3537                 if (!r) {
3538                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
3539                         r = amdgpu_ib_ring_tests(tmp_adev);
3540                         if (r) {
3541                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
3542                                 r = amdgpu_device_ip_suspend(tmp_adev);
3543                                 need_full_reset = true;
3544                                 r = -EAGAIN;
3545                                 goto end;
3546                         }
3547                 }
3548
3549                 if (!r)
3550                         r = amdgpu_device_recover_vram(tmp_adev);
3551                 else
3552                         tmp_adev->asic_reset_res = r;
3553         }
3554
3555 end:
3556         *need_full_reset_arg = need_full_reset;
3557         return r;
3558 }
3559
3560 static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, bool trylock)
3561 {
3562         if (trylock) {
3563                 if (!mutex_trylock(&adev->lock_reset))
3564                         return false;
3565         } else
3566                 mutex_lock(&adev->lock_reset);
3567
3568         atomic_inc(&adev->gpu_reset_counter);
3569         adev->in_gpu_reset = 1;
3570         /* Block kfd: SRIOV would do it separately */
3571         if (!amdgpu_sriov_vf(adev))
3572                 amdgpu_amdkfd_pre_reset(adev);
3573
3574         return true;
3575 }
3576
3577 static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
3578 {
3579         /*unlock kfd: SRIOV would do it separately */
3580         if (!amdgpu_sriov_vf(adev))
3581                 amdgpu_amdkfd_post_reset(adev);
3582         amdgpu_vf_error_trans_all(adev);
3583         adev->in_gpu_reset = 0;
3584         mutex_unlock(&adev->lock_reset);
3585 }
3586
3587
3588 /**
3589  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
3590  *
3591  * @adev: amdgpu device pointer
3592  * @job: which job trigger hang
3593  *
3594  * Attempt to reset the GPU if it has hung (all asics).
3595  * Attempt to do soft-reset or full-reset and reinitialize Asic
3596  * Returns 0 for success or an error on failure.
3597  */
3598
3599 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
3600                               struct amdgpu_job *job)
3601 {
3602         struct list_head device_list, *device_list_handle =  NULL;
3603         bool need_full_reset, job_signaled;
3604         struct amdgpu_hive_info *hive = NULL;
3605         struct amdgpu_device *tmp_adev = NULL;
3606         int i, r = 0;
3607
3608         need_full_reset = job_signaled = false;
3609         INIT_LIST_HEAD(&device_list);
3610
3611         dev_info(adev->dev, "GPU reset begin!\n");
3612
3613         cancel_delayed_work_sync(&adev->delayed_init_work);
3614
3615         hive = amdgpu_get_xgmi_hive(adev, false);
3616
3617         /*
3618          * Here we trylock to avoid chain of resets executing from
3619          * either trigger by jobs on different adevs in XGMI hive or jobs on
3620          * different schedulers for same device while this TO handler is running.
3621          * We always reset all schedulers for device and all devices for XGMI
3622          * hive so that should take care of them too.
3623          */
3624
3625         if (hive && !mutex_trylock(&hive->reset_lock)) {
3626                 DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
3627                          job->base.id, hive->hive_id);
3628                 return 0;
3629         }
3630
3631         /* Start with adev pre asic reset first for soft reset check.*/
3632         if (!amdgpu_device_lock_adev(adev, !hive)) {
3633                 DRM_INFO("Bailing on TDR for s_job:%llx, as another already in progress",
3634                                          job->base.id);
3635                 return 0;
3636         }
3637
3638         /* Build list of devices to reset */
3639         if  (adev->gmc.xgmi.num_physical_nodes > 1) {
3640                 if (!hive) {
3641                         amdgpu_device_unlock_adev(adev);
3642                         return -ENODEV;
3643                 }
3644
3645                 /*
3646                  * In case we are in XGMI hive mode device reset is done for all the
3647                  * nodes in the hive to retrain all XGMI links and hence the reset
3648                  * sequence is executed in loop on all nodes.
3649                  */
3650                 device_list_handle = &hive->device_list;
3651         } else {
3652                 list_add_tail(&adev->gmc.xgmi.head, &device_list);
3653                 device_list_handle = &device_list;
3654         }
3655
3656         /* block all schedulers and reset given job's ring */
3657         list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3658                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3659                         struct amdgpu_ring *ring = tmp_adev->rings[i];
3660
3661                         if (!ring || !ring->sched.thread)
3662                                 continue;
3663
3664                         drm_sched_stop(&ring->sched, &job->base);
3665                 }
3666         }
3667
3668
3669         /*
3670          * Must check guilty signal here since after this point all old
3671          * HW fences are force signaled.
3672          *
3673          * job->base holds a reference to parent fence
3674          */
3675         if (job && job->base.s_fence->parent &&
3676             dma_fence_is_signaled(job->base.s_fence->parent))
3677                 job_signaled = true;
3678
3679         if (!amdgpu_device_ip_need_full_reset(adev))
3680                 device_list_handle = &device_list;
3681
3682         if (job_signaled) {
3683                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
3684                 goto skip_hw_reset;
3685         }
3686
3687
3688         /* Guilty job will be freed after this*/
3689         r = amdgpu_device_pre_asic_reset(adev,
3690                                          job,
3691                                          &need_full_reset);
3692         if (r) {
3693                 /*TODO Should we stop ?*/
3694                 DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
3695                           r, adev->ddev->unique);
3696                 adev->asic_reset_res = r;
3697         }
3698
3699 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
3700         list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3701
3702                 if (tmp_adev == adev)
3703                         continue;
3704
3705                 amdgpu_device_lock_adev(tmp_adev, false);
3706                 r = amdgpu_device_pre_asic_reset(tmp_adev,
3707                                                  NULL,
3708                                                  &need_full_reset);
3709                 /*TODO Should we stop ?*/
3710                 if (r) {
3711                         DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
3712                                   r, tmp_adev->ddev->unique);
3713                         tmp_adev->asic_reset_res = r;
3714                 }
3715         }
3716
3717         /* Actual ASIC resets if needed.*/
3718         /* TODO Implement XGMI hive reset logic for SRIOV */
3719         if (amdgpu_sriov_vf(adev)) {
3720                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
3721                 if (r)
3722                         adev->asic_reset_res = r;
3723         } else {
3724                 r  = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset);
3725                 if (r && r == -EAGAIN)
3726                         goto retry;
3727         }
3728
3729 skip_hw_reset:
3730
3731         /* Post ASIC reset for all devs .*/
3732         list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3733                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3734                         struct amdgpu_ring *ring = tmp_adev->rings[i];
3735
3736                         if (!ring || !ring->sched.thread)
3737                                 continue;
3738
3739                         /* No point to resubmit jobs if we didn't HW reset*/
3740                         if (!tmp_adev->asic_reset_res && !job_signaled)
3741                                 drm_sched_resubmit_jobs(&ring->sched);
3742
3743                         drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
3744                 }
3745
3746                 if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
3747                         drm_helper_resume_force_mode(tmp_adev->ddev);
3748                 }
3749
3750                 tmp_adev->asic_reset_res = 0;
3751
3752                 if (r) {
3753                         /* bad news, how to tell it to userspace ? */
3754                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
3755                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
3756                 } else {
3757                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&adev->gpu_reset_counter));
3758                 }
3759
3760                 amdgpu_device_unlock_adev(tmp_adev);
3761         }
3762
3763         if (hive)
3764                 mutex_unlock(&hive->reset_lock);
3765
3766         if (r)
3767                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
3768         return r;
3769 }
3770
3771 /**
3772  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
3773  *
3774  * @adev: amdgpu_device pointer
3775  *
3776  * Fetchs and stores in the driver the PCIE capabilities (gen speed
3777  * and lanes) of the slot the device is in. Handles APUs and
3778  * virtualized environments where PCIE config space may not be available.
3779  */
3780 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
3781 {
3782         struct pci_dev *pdev;
3783         enum pci_bus_speed speed_cap, platform_speed_cap;
3784         enum pcie_link_width platform_link_width;
3785
3786         if (amdgpu_pcie_gen_cap)
3787                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
3788
3789         if (amdgpu_pcie_lane_cap)
3790                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
3791
3792         /* covers APUs as well */
3793         if (pci_is_root_bus(adev->pdev->bus)) {
3794                 if (adev->pm.pcie_gen_mask == 0)
3795                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3796                 if (adev->pm.pcie_mlw_mask == 0)
3797                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3798                 return;
3799         }
3800
3801         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
3802                 return;
3803
3804         pcie_bandwidth_available(adev->pdev, NULL,
3805                                  &platform_speed_cap, &platform_link_width);
3806
3807         if (adev->pm.pcie_gen_mask == 0) {
3808                 /* asic caps */
3809                 pdev = adev->pdev;
3810                 speed_cap = pcie_get_speed_cap(pdev);
3811                 if (speed_cap == PCI_SPEED_UNKNOWN) {
3812                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3813                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3814                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
3815                 } else {
3816                         if (speed_cap == PCIE_SPEED_16_0GT)
3817                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3818                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3819                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
3820                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
3821                         else if (speed_cap == PCIE_SPEED_8_0GT)
3822                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3823                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3824                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
3825                         else if (speed_cap == PCIE_SPEED_5_0GT)
3826                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3827                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
3828                         else
3829                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
3830                 }
3831                 /* platform caps */
3832                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
3833                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3834                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
3835                 } else {
3836                         if (platform_speed_cap == PCIE_SPEED_16_0GT)
3837                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3838                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3839                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
3840                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
3841                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
3842                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3843                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3844                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
3845                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
3846                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3847                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
3848                         else
3849                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
3850
3851                 }
3852         }
3853         if (adev->pm.pcie_mlw_mask == 0) {
3854                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
3855                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
3856                 } else {
3857                         switch (platform_link_width) {
3858                         case PCIE_LNK_X32:
3859                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
3860                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3861                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3862                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3863                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3864                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3865                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3866                                 break;
3867                         case PCIE_LNK_X16:
3868                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3869                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3870                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3871                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3872                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3873                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3874                                 break;
3875                         case PCIE_LNK_X12:
3876                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3877                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3878                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3879                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3880                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3881                                 break;
3882                         case PCIE_LNK_X8:
3883                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3884                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3885                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3886                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3887                                 break;
3888                         case PCIE_LNK_X4:
3889                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3890                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3891                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3892                                 break;
3893                         case PCIE_LNK_X2:
3894                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3895                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3896                                 break;
3897                         case PCIE_LNK_X1:
3898                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
3899                                 break;
3900                         default:
3901                                 break;
3902                         }
3903                 }
3904         }
3905 }
3906