]> Git Repo - linux.git/blob - drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
drm/amdgpu - Disable all irqs before disabling all CRTCs
[linux.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_device.c
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/kthread.h>
29 #include <linux/console.h>
30 #include <linux/slab.h>
31 #include <drm/drmP.h>
32 #include <drm/drm_crtc_helper.h>
33 #include <drm/drm_atomic_helper.h>
34 #include <drm/amdgpu_drm.h>
35 #include <linux/vgaarb.h>
36 #include <linux/vga_switcheroo.h>
37 #include <linux/efi.h>
38 #include "amdgpu.h"
39 #include "amdgpu_trace.h"
40 #include "amdgpu_i2c.h"
41 #include "atom.h"
42 #include "amdgpu_atombios.h"
43 #include "amdgpu_atomfirmware.h"
44 #include "amd_pcie.h"
45 #ifdef CONFIG_DRM_AMDGPU_SI
46 #include "si.h"
47 #endif
48 #ifdef CONFIG_DRM_AMDGPU_CIK
49 #include "cik.h"
50 #endif
51 #include "vi.h"
52 #include "soc15.h"
53 #include "bif/bif_4_1_d.h"
54 #include <linux/pci.h>
55 #include <linux/firmware.h>
56 #include "amdgpu_vf_error.h"
57
58 #include "amdgpu_amdkfd.h"
59 #include "amdgpu_pm.h"
60
61 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
62 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
63
64 #define AMDGPU_RESUME_MS                2000
65
66 static const char *amdgpu_asic_name[] = {
67         "TAHITI",
68         "PITCAIRN",
69         "VERDE",
70         "OLAND",
71         "HAINAN",
72         "BONAIRE",
73         "KAVERI",
74         "KABINI",
75         "HAWAII",
76         "MULLINS",
77         "TOPAZ",
78         "TONGA",
79         "FIJI",
80         "CARRIZO",
81         "STONEY",
82         "POLARIS10",
83         "POLARIS11",
84         "POLARIS12",
85         "VEGA10",
86         "RAVEN",
87         "LAST",
88 };
89
90 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
91
92 /**
93  * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control
94  *
95  * @dev: drm_device pointer
96  *
97  * Returns true if the device is a dGPU with HG/PX power control,
98  * otherwise return false.
99  */
100 bool amdgpu_device_is_px(struct drm_device *dev)
101 {
102         struct amdgpu_device *adev = dev->dev_private;
103
104         if (adev->flags & AMD_IS_PX)
105                 return true;
106         return false;
107 }
108
109 /*
110  * MMIO register access helper functions.
111  */
112 /**
113  * amdgpu_mm_rreg - read a memory mapped IO register
114  *
115  * @adev: amdgpu_device pointer
116  * @reg: dword aligned register offset
117  * @acc_flags: access flags which require special behavior
118  *
119  * Returns the 32 bit value from the offset specified.
120  */
121 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
122                         uint32_t acc_flags)
123 {
124         uint32_t ret;
125
126         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
127                 return amdgpu_virt_kiq_rreg(adev, reg);
128
129         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
130                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
131         else {
132                 unsigned long flags;
133
134                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
135                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
136                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
137                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
138         }
139         trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
140         return ret;
141 }
142
143 /*
144  * MMIO register read with bytes helper functions
145  * @offset:bytes offset from MMIO start
146  *
147 */
148
149 /**
150  * amdgpu_mm_rreg8 - read a memory mapped IO register
151  *
152  * @adev: amdgpu_device pointer
153  * @offset: byte aligned register offset
154  *
155  * Returns the 8 bit value from the offset specified.
156  */
157 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
158         if (offset < adev->rmmio_size)
159                 return (readb(adev->rmmio + offset));
160         BUG();
161 }
162
163 /*
164  * MMIO register write with bytes helper functions
165  * @offset:bytes offset from MMIO start
166  * @value: the value want to be written to the register
167  *
168 */
169 /**
170  * amdgpu_mm_wreg8 - read a memory mapped IO register
171  *
172  * @adev: amdgpu_device pointer
173  * @offset: byte aligned register offset
174  * @value: 8 bit value to write
175  *
176  * Writes the value specified to the offset specified.
177  */
178 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
179         if (offset < adev->rmmio_size)
180                 writeb(value, adev->rmmio + offset);
181         else
182                 BUG();
183 }
184
185 /**
186  * amdgpu_mm_wreg - write to a memory mapped IO register
187  *
188  * @adev: amdgpu_device pointer
189  * @reg: dword aligned register offset
190  * @v: 32 bit value to write to the register
191  * @acc_flags: access flags which require special behavior
192  *
193  * Writes the value specified to the offset specified.
194  */
195 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
196                     uint32_t acc_flags)
197 {
198         trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
199
200         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
201                 adev->last_mm_index = v;
202         }
203
204         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
205                 return amdgpu_virt_kiq_wreg(adev, reg, v);
206
207         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
208                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
209         else {
210                 unsigned long flags;
211
212                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
213                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
214                 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
215                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
216         }
217
218         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
219                 udelay(500);
220         }
221 }
222
223 /**
224  * amdgpu_io_rreg - read an IO register
225  *
226  * @adev: amdgpu_device pointer
227  * @reg: dword aligned register offset
228  *
229  * Returns the 32 bit value from the offset specified.
230  */
231 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
232 {
233         if ((reg * 4) < adev->rio_mem_size)
234                 return ioread32(adev->rio_mem + (reg * 4));
235         else {
236                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
237                 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
238         }
239 }
240
241 /**
242  * amdgpu_io_wreg - write to an IO register
243  *
244  * @adev: amdgpu_device pointer
245  * @reg: dword aligned register offset
246  * @v: 32 bit value to write to the register
247  *
248  * Writes the value specified to the offset specified.
249  */
250 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
251 {
252         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
253                 adev->last_mm_index = v;
254         }
255
256         if ((reg * 4) < adev->rio_mem_size)
257                 iowrite32(v, adev->rio_mem + (reg * 4));
258         else {
259                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
260                 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
261         }
262
263         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
264                 udelay(500);
265         }
266 }
267
268 /**
269  * amdgpu_mm_rdoorbell - read a doorbell dword
270  *
271  * @adev: amdgpu_device pointer
272  * @index: doorbell index
273  *
274  * Returns the value in the doorbell aperture at the
275  * requested doorbell index (CIK).
276  */
277 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
278 {
279         if (index < adev->doorbell.num_doorbells) {
280                 return readl(adev->doorbell.ptr + index);
281         } else {
282                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
283                 return 0;
284         }
285 }
286
287 /**
288  * amdgpu_mm_wdoorbell - write a doorbell dword
289  *
290  * @adev: amdgpu_device pointer
291  * @index: doorbell index
292  * @v: value to write
293  *
294  * Writes @v to the doorbell aperture at the
295  * requested doorbell index (CIK).
296  */
297 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
298 {
299         if (index < adev->doorbell.num_doorbells) {
300                 writel(v, adev->doorbell.ptr + index);
301         } else {
302                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
303         }
304 }
305
306 /**
307  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
308  *
309  * @adev: amdgpu_device pointer
310  * @index: doorbell index
311  *
312  * Returns the value in the doorbell aperture at the
313  * requested doorbell index (VEGA10+).
314  */
315 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
316 {
317         if (index < adev->doorbell.num_doorbells) {
318                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
319         } else {
320                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
321                 return 0;
322         }
323 }
324
325 /**
326  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
327  *
328  * @adev: amdgpu_device pointer
329  * @index: doorbell index
330  * @v: value to write
331  *
332  * Writes @v to the doorbell aperture at the
333  * requested doorbell index (VEGA10+).
334  */
335 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
336 {
337         if (index < adev->doorbell.num_doorbells) {
338                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
339         } else {
340                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
341         }
342 }
343
344 /**
345  * amdgpu_invalid_rreg - dummy reg read function
346  *
347  * @adev: amdgpu device pointer
348  * @reg: offset of register
349  *
350  * Dummy register read function.  Used for register blocks
351  * that certain asics don't have (all asics).
352  * Returns the value in the register.
353  */
354 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
355 {
356         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
357         BUG();
358         return 0;
359 }
360
361 /**
362  * amdgpu_invalid_wreg - dummy reg write function
363  *
364  * @adev: amdgpu device pointer
365  * @reg: offset of register
366  * @v: value to write to the register
367  *
368  * Dummy register read function.  Used for register blocks
369  * that certain asics don't have (all asics).
370  */
371 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
372 {
373         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
374                   reg, v);
375         BUG();
376 }
377
378 /**
379  * amdgpu_block_invalid_rreg - dummy reg read function
380  *
381  * @adev: amdgpu device pointer
382  * @block: offset of instance
383  * @reg: offset of register
384  *
385  * Dummy register read function.  Used for register blocks
386  * that certain asics don't have (all asics).
387  * Returns the value in the register.
388  */
389 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
390                                           uint32_t block, uint32_t reg)
391 {
392         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
393                   reg, block);
394         BUG();
395         return 0;
396 }
397
398 /**
399  * amdgpu_block_invalid_wreg - dummy reg write function
400  *
401  * @adev: amdgpu device pointer
402  * @block: offset of instance
403  * @reg: offset of register
404  * @v: value to write to the register
405  *
406  * Dummy register read function.  Used for register blocks
407  * that certain asics don't have (all asics).
408  */
409 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
410                                       uint32_t block,
411                                       uint32_t reg, uint32_t v)
412 {
413         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
414                   reg, block, v);
415         BUG();
416 }
417
418 /**
419  * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
420  *
421  * @adev: amdgpu device pointer
422  *
423  * Allocates a scratch page of VRAM for use by various things in the
424  * driver.
425  */
426 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
427 {
428         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
429                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
430                                        &adev->vram_scratch.robj,
431                                        &adev->vram_scratch.gpu_addr,
432                                        (void **)&adev->vram_scratch.ptr);
433 }
434
435 /**
436  * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
437  *
438  * @adev: amdgpu device pointer
439  *
440  * Frees the VRAM scratch page.
441  */
442 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
443 {
444         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
445 }
446
447 /**
448  * amdgpu_device_program_register_sequence - program an array of registers.
449  *
450  * @adev: amdgpu_device pointer
451  * @registers: pointer to the register array
452  * @array_size: size of the register array
453  *
454  * Programs an array or registers with and and or masks.
455  * This is a helper for setting golden registers.
456  */
457 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
458                                              const u32 *registers,
459                                              const u32 array_size)
460 {
461         u32 tmp, reg, and_mask, or_mask;
462         int i;
463
464         if (array_size % 3)
465                 return;
466
467         for (i = 0; i < array_size; i +=3) {
468                 reg = registers[i + 0];
469                 and_mask = registers[i + 1];
470                 or_mask = registers[i + 2];
471
472                 if (and_mask == 0xffffffff) {
473                         tmp = or_mask;
474                 } else {
475                         tmp = RREG32(reg);
476                         tmp &= ~and_mask;
477                         tmp |= or_mask;
478                 }
479                 WREG32(reg, tmp);
480         }
481 }
482
483 /**
484  * amdgpu_device_pci_config_reset - reset the GPU
485  *
486  * @adev: amdgpu_device pointer
487  *
488  * Resets the GPU using the pci config reset sequence.
489  * Only applicable to asics prior to vega10.
490  */
491 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
492 {
493         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
494 }
495
496 /*
497  * GPU doorbell aperture helpers function.
498  */
499 /**
500  * amdgpu_device_doorbell_init - Init doorbell driver information.
501  *
502  * @adev: amdgpu_device pointer
503  *
504  * Init doorbell driver information (CIK)
505  * Returns 0 on success, error on failure.
506  */
507 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
508 {
509         /* No doorbell on SI hardware generation */
510         if (adev->asic_type < CHIP_BONAIRE) {
511                 adev->doorbell.base = 0;
512                 adev->doorbell.size = 0;
513                 adev->doorbell.num_doorbells = 0;
514                 adev->doorbell.ptr = NULL;
515                 return 0;
516         }
517
518         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
519                 return -EINVAL;
520
521         /* doorbell bar mapping */
522         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
523         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
524
525         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
526                                              AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
527         if (adev->doorbell.num_doorbells == 0)
528                 return -EINVAL;
529
530         adev->doorbell.ptr = ioremap(adev->doorbell.base,
531                                      adev->doorbell.num_doorbells *
532                                      sizeof(u32));
533         if (adev->doorbell.ptr == NULL)
534                 return -ENOMEM;
535
536         return 0;
537 }
538
539 /**
540  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
541  *
542  * @adev: amdgpu_device pointer
543  *
544  * Tear down doorbell driver information (CIK)
545  */
546 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
547 {
548         iounmap(adev->doorbell.ptr);
549         adev->doorbell.ptr = NULL;
550 }
551
552
553
554 /*
555  * amdgpu_device_wb_*()
556  * Writeback is the method by which the GPU updates special pages in memory
557  * with the status of certain GPU events (fences, ring pointers,etc.).
558  */
559
560 /**
561  * amdgpu_device_wb_fini - Disable Writeback and free memory
562  *
563  * @adev: amdgpu_device pointer
564  *
565  * Disables Writeback and frees the Writeback memory (all asics).
566  * Used at driver shutdown.
567  */
568 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
569 {
570         if (adev->wb.wb_obj) {
571                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
572                                       &adev->wb.gpu_addr,
573                                       (void **)&adev->wb.wb);
574                 adev->wb.wb_obj = NULL;
575         }
576 }
577
578 /**
579  * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
580  *
581  * @adev: amdgpu_device pointer
582  *
583  * Initializes writeback and allocates writeback memory (all asics).
584  * Used at driver startup.
585  * Returns 0 on success or an -error on failure.
586  */
587 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
588 {
589         int r;
590
591         if (adev->wb.wb_obj == NULL) {
592                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
593                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
594                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
595                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
596                                             (void **)&adev->wb.wb);
597                 if (r) {
598                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
599                         return r;
600                 }
601
602                 adev->wb.num_wb = AMDGPU_MAX_WB;
603                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
604
605                 /* clear wb memory */
606                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
607         }
608
609         return 0;
610 }
611
612 /**
613  * amdgpu_device_wb_get - Allocate a wb entry
614  *
615  * @adev: amdgpu_device pointer
616  * @wb: wb index
617  *
618  * Allocate a wb slot for use by the driver (all asics).
619  * Returns 0 on success or -EINVAL on failure.
620  */
621 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
622 {
623         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
624
625         if (offset < adev->wb.num_wb) {
626                 __set_bit(offset, adev->wb.used);
627                 *wb = offset << 3; /* convert to dw offset */
628                 return 0;
629         } else {
630                 return -EINVAL;
631         }
632 }
633
634 /**
635  * amdgpu_device_wb_free - Free a wb entry
636  *
637  * @adev: amdgpu_device pointer
638  * @wb: wb index
639  *
640  * Free a wb slot allocated for use by the driver (all asics)
641  */
642 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
643 {
644         wb >>= 3;
645         if (wb < adev->wb.num_wb)
646                 __clear_bit(wb, adev->wb.used);
647 }
648
649 /**
650  * amdgpu_device_vram_location - try to find VRAM location
651  *
652  * @adev: amdgpu device structure holding all necessary informations
653  * @mc: memory controller structure holding memory informations
654  * @base: base address at which to put VRAM
655  *
656  * Function will try to place VRAM at base address provided
657  * as parameter.
658  */
659 void amdgpu_device_vram_location(struct amdgpu_device *adev,
660                                  struct amdgpu_gmc *mc, u64 base)
661 {
662         uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
663
664         mc->vram_start = base;
665         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
666         if (limit && limit < mc->real_vram_size)
667                 mc->real_vram_size = limit;
668         dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
669                         mc->mc_vram_size >> 20, mc->vram_start,
670                         mc->vram_end, mc->real_vram_size >> 20);
671 }
672
673 /**
674  * amdgpu_device_gart_location - try to find GTT location
675  *
676  * @adev: amdgpu device structure holding all necessary informations
677  * @mc: memory controller structure holding memory informations
678  *
679  * Function will place try to place GTT before or after VRAM.
680  *
681  * If GTT size is bigger than space left then we ajust GTT size.
682  * Thus function will never fails.
683  *
684  * FIXME: when reducing GTT size align new size on power of 2.
685  */
686 void amdgpu_device_gart_location(struct amdgpu_device *adev,
687                                  struct amdgpu_gmc *mc)
688 {
689         u64 size_af, size_bf;
690
691         size_af = adev->gmc.mc_mask - mc->vram_end;
692         size_bf = mc->vram_start;
693         if (size_bf > size_af) {
694                 if (mc->gart_size > size_bf) {
695                         dev_warn(adev->dev, "limiting GTT\n");
696                         mc->gart_size = size_bf;
697                 }
698                 mc->gart_start = 0;
699         } else {
700                 if (mc->gart_size > size_af) {
701                         dev_warn(adev->dev, "limiting GTT\n");
702                         mc->gart_size = size_af;
703                 }
704                 /* VCE doesn't like it when BOs cross a 4GB segment, so align
705                  * the GART base on a 4GB boundary as well.
706                  */
707                 mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL);
708         }
709         mc->gart_end = mc->gart_start + mc->gart_size - 1;
710         dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
711                         mc->gart_size >> 20, mc->gart_start, mc->gart_end);
712 }
713
714 /**
715  * amdgpu_device_resize_fb_bar - try to resize FB BAR
716  *
717  * @adev: amdgpu_device pointer
718  *
719  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
720  * to fail, but if any of the BARs is not accessible after the size we abort
721  * driver loading by returning -ENODEV.
722  */
723 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
724 {
725         u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
726         u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
727         struct pci_bus *root;
728         struct resource *res;
729         unsigned i;
730         u16 cmd;
731         int r;
732
733         /* Bypass for VF */
734         if (amdgpu_sriov_vf(adev))
735                 return 0;
736
737         /* Check if the root BUS has 64bit memory resources */
738         root = adev->pdev->bus;
739         while (root->parent)
740                 root = root->parent;
741
742         pci_bus_for_each_resource(root, res, i) {
743                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
744                     res->start > 0x100000000ull)
745                         break;
746         }
747
748         /* Trying to resize is pointless without a root hub window above 4GB */
749         if (!res)
750                 return 0;
751
752         /* Disable memory decoding while we change the BAR addresses and size */
753         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
754         pci_write_config_word(adev->pdev, PCI_COMMAND,
755                               cmd & ~PCI_COMMAND_MEMORY);
756
757         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
758         amdgpu_device_doorbell_fini(adev);
759         if (adev->asic_type >= CHIP_BONAIRE)
760                 pci_release_resource(adev->pdev, 2);
761
762         pci_release_resource(adev->pdev, 0);
763
764         r = pci_resize_resource(adev->pdev, 0, rbar_size);
765         if (r == -ENOSPC)
766                 DRM_INFO("Not enough PCI address space for a large BAR.");
767         else if (r && r != -ENOTSUPP)
768                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
769
770         pci_assign_unassigned_bus_resources(adev->pdev->bus);
771
772         /* When the doorbell or fb BAR isn't available we have no chance of
773          * using the device.
774          */
775         r = amdgpu_device_doorbell_init(adev);
776         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
777                 return -ENODEV;
778
779         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
780
781         return 0;
782 }
783
784 /*
785  * GPU helpers function.
786  */
787 /**
788  * amdgpu_device_need_post - check if the hw need post or not
789  *
790  * @adev: amdgpu_device pointer
791  *
792  * Check if the asic has been initialized (all asics) at driver startup
793  * or post is needed if  hw reset is performed.
794  * Returns true if need or false if not.
795  */
796 bool amdgpu_device_need_post(struct amdgpu_device *adev)
797 {
798         uint32_t reg;
799
800         if (amdgpu_sriov_vf(adev))
801                 return false;
802
803         if (amdgpu_passthrough(adev)) {
804                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
805                  * some old smc fw still need driver do vPost otherwise gpu hang, while
806                  * those smc fw version above 22.15 doesn't have this flaw, so we force
807                  * vpost executed for smc version below 22.15
808                  */
809                 if (adev->asic_type == CHIP_FIJI) {
810                         int err;
811                         uint32_t fw_ver;
812                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
813                         /* force vPost if error occured */
814                         if (err)
815                                 return true;
816
817                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
818                         if (fw_ver < 0x00160e00)
819                                 return true;
820                 }
821         }
822
823         if (adev->has_hw_reset) {
824                 adev->has_hw_reset = false;
825                 return true;
826         }
827
828         /* bios scratch used on CIK+ */
829         if (adev->asic_type >= CHIP_BONAIRE)
830                 return amdgpu_atombios_scratch_need_asic_init(adev);
831
832         /* check MEM_SIZE for older asics */
833         reg = amdgpu_asic_get_config_memsize(adev);
834
835         if ((reg != 0) && (reg != 0xffffffff))
836                 return false;
837
838         return true;
839 }
840
841 /* if we get transitioned to only one device, take VGA back */
842 /**
843  * amdgpu_device_vga_set_decode - enable/disable vga decode
844  *
845  * @cookie: amdgpu_device pointer
846  * @state: enable/disable vga decode
847  *
848  * Enable/disable vga decode (all asics).
849  * Returns VGA resource flags.
850  */
851 static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
852 {
853         struct amdgpu_device *adev = cookie;
854         amdgpu_asic_set_vga_state(adev, state);
855         if (state)
856                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
857                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
858         else
859                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
860 }
861
862 /**
863  * amdgpu_device_check_block_size - validate the vm block size
864  *
865  * @adev: amdgpu_device pointer
866  *
867  * Validates the vm block size specified via module parameter.
868  * The vm block size defines number of bits in page table versus page directory,
869  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
870  * page table and the remaining bits are in the page directory.
871  */
872 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
873 {
874         /* defines number of bits in page table versus page directory,
875          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
876          * page table and the remaining bits are in the page directory */
877         if (amdgpu_vm_block_size == -1)
878                 return;
879
880         if (amdgpu_vm_block_size < 9) {
881                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
882                          amdgpu_vm_block_size);
883                 amdgpu_vm_block_size = -1;
884         }
885 }
886
887 /**
888  * amdgpu_device_check_vm_size - validate the vm size
889  *
890  * @adev: amdgpu_device pointer
891  *
892  * Validates the vm size in GB specified via module parameter.
893  * The VM size is the size of the GPU virtual memory space in GB.
894  */
895 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
896 {
897         /* no need to check the default value */
898         if (amdgpu_vm_size == -1)
899                 return;
900
901         if (amdgpu_vm_size < 1) {
902                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
903                          amdgpu_vm_size);
904                 amdgpu_vm_size = -1;
905         }
906 }
907
908 /**
909  * amdgpu_device_check_arguments - validate module params
910  *
911  * @adev: amdgpu_device pointer
912  *
913  * Validates certain module parameters and updates
914  * the associated values used by the driver (all asics).
915  */
916 static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
917 {
918         if (amdgpu_sched_jobs < 4) {
919                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
920                          amdgpu_sched_jobs);
921                 amdgpu_sched_jobs = 4;
922         } else if (!is_power_of_2(amdgpu_sched_jobs)){
923                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
924                          amdgpu_sched_jobs);
925                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
926         }
927
928         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
929                 /* gart size must be greater or equal to 32M */
930                 dev_warn(adev->dev, "gart size (%d) too small\n",
931                          amdgpu_gart_size);
932                 amdgpu_gart_size = -1;
933         }
934
935         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
936                 /* gtt size must be greater or equal to 32M */
937                 dev_warn(adev->dev, "gtt size (%d) too small\n",
938                                  amdgpu_gtt_size);
939                 amdgpu_gtt_size = -1;
940         }
941
942         /* valid range is between 4 and 9 inclusive */
943         if (amdgpu_vm_fragment_size != -1 &&
944             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
945                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
946                 amdgpu_vm_fragment_size = -1;
947         }
948
949         amdgpu_device_check_vm_size(adev);
950
951         amdgpu_device_check_block_size(adev);
952
953         if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
954             !is_power_of_2(amdgpu_vram_page_split))) {
955                 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
956                          amdgpu_vram_page_split);
957                 amdgpu_vram_page_split = 1024;
958         }
959
960         if (amdgpu_lockup_timeout == 0) {
961                 dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n");
962                 amdgpu_lockup_timeout = 10000;
963         }
964
965         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
966 }
967
968 /**
969  * amdgpu_switcheroo_set_state - set switcheroo state
970  *
971  * @pdev: pci dev pointer
972  * @state: vga_switcheroo state
973  *
974  * Callback for the switcheroo driver.  Suspends or resumes the
975  * the asics before or after it is powered up using ACPI methods.
976  */
977 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
978 {
979         struct drm_device *dev = pci_get_drvdata(pdev);
980
981         if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
982                 return;
983
984         if (state == VGA_SWITCHEROO_ON) {
985                 pr_info("amdgpu: switched on\n");
986                 /* don't suspend or resume card normally */
987                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
988
989                 amdgpu_device_resume(dev, true, true);
990
991                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
992                 drm_kms_helper_poll_enable(dev);
993         } else {
994                 pr_info("amdgpu: switched off\n");
995                 drm_kms_helper_poll_disable(dev);
996                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
997                 amdgpu_device_suspend(dev, true, true);
998                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
999         }
1000 }
1001
1002 /**
1003  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1004  *
1005  * @pdev: pci dev pointer
1006  *
1007  * Callback for the switcheroo driver.  Check of the switcheroo
1008  * state can be changed.
1009  * Returns true if the state can be changed, false if not.
1010  */
1011 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1012 {
1013         struct drm_device *dev = pci_get_drvdata(pdev);
1014
1015         /*
1016         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1017         * locking inversion with the driver load path. And the access here is
1018         * completely racy anyway. So don't bother with locking for now.
1019         */
1020         return dev->open_count == 0;
1021 }
1022
1023 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1024         .set_gpu_state = amdgpu_switcheroo_set_state,
1025         .reprobe = NULL,
1026         .can_switch = amdgpu_switcheroo_can_switch,
1027 };
1028
1029 /**
1030  * amdgpu_device_ip_set_clockgating_state - set the CG state
1031  *
1032  * @adev: amdgpu_device pointer
1033  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1034  * @state: clockgating state (gate or ungate)
1035  *
1036  * Sets the requested clockgating state for all instances of
1037  * the hardware IP specified.
1038  * Returns the error code from the last instance.
1039  */
1040 int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev,
1041                                            enum amd_ip_block_type block_type,
1042                                            enum amd_clockgating_state state)
1043 {
1044         int i, r = 0;
1045
1046         for (i = 0; i < adev->num_ip_blocks; i++) {
1047                 if (!adev->ip_blocks[i].status.valid)
1048                         continue;
1049                 if (adev->ip_blocks[i].version->type != block_type)
1050                         continue;
1051                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1052                         continue;
1053                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1054                         (void *)adev, state);
1055                 if (r)
1056                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1057                                   adev->ip_blocks[i].version->funcs->name, r);
1058         }
1059         return r;
1060 }
1061
1062 /**
1063  * amdgpu_device_ip_set_powergating_state - set the PG state
1064  *
1065  * @adev: amdgpu_device pointer
1066  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1067  * @state: powergating state (gate or ungate)
1068  *
1069  * Sets the requested powergating state for all instances of
1070  * the hardware IP specified.
1071  * Returns the error code from the last instance.
1072  */
1073 int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev,
1074                                            enum amd_ip_block_type block_type,
1075                                            enum amd_powergating_state state)
1076 {
1077         int i, r = 0;
1078
1079         for (i = 0; i < adev->num_ip_blocks; i++) {
1080                 if (!adev->ip_blocks[i].status.valid)
1081                         continue;
1082                 if (adev->ip_blocks[i].version->type != block_type)
1083                         continue;
1084                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1085                         continue;
1086                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1087                         (void *)adev, state);
1088                 if (r)
1089                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1090                                   adev->ip_blocks[i].version->funcs->name, r);
1091         }
1092         return r;
1093 }
1094
1095 /**
1096  * amdgpu_device_ip_get_clockgating_state - get the CG state
1097  *
1098  * @adev: amdgpu_device pointer
1099  * @flags: clockgating feature flags
1100  *
1101  * Walks the list of IPs on the device and updates the clockgating
1102  * flags for each IP.
1103  * Updates @flags with the feature flags for each hardware IP where
1104  * clockgating is enabled.
1105  */
1106 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1107                                             u32 *flags)
1108 {
1109         int i;
1110
1111         for (i = 0; i < adev->num_ip_blocks; i++) {
1112                 if (!adev->ip_blocks[i].status.valid)
1113                         continue;
1114                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1115                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1116         }
1117 }
1118
1119 /**
1120  * amdgpu_device_ip_wait_for_idle - wait for idle
1121  *
1122  * @adev: amdgpu_device pointer
1123  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1124  *
1125  * Waits for the request hardware IP to be idle.
1126  * Returns 0 for success or a negative error code on failure.
1127  */
1128 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1129                                    enum amd_ip_block_type block_type)
1130 {
1131         int i, r;
1132
1133         for (i = 0; i < adev->num_ip_blocks; i++) {
1134                 if (!adev->ip_blocks[i].status.valid)
1135                         continue;
1136                 if (adev->ip_blocks[i].version->type == block_type) {
1137                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1138                         if (r)
1139                                 return r;
1140                         break;
1141                 }
1142         }
1143         return 0;
1144
1145 }
1146
1147 /**
1148  * amdgpu_device_ip_is_idle - is the hardware IP idle
1149  *
1150  * @adev: amdgpu_device pointer
1151  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1152  *
1153  * Check if the hardware IP is idle or not.
1154  * Returns true if it the IP is idle, false if not.
1155  */
1156 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1157                               enum amd_ip_block_type block_type)
1158 {
1159         int i;
1160
1161         for (i = 0; i < adev->num_ip_blocks; i++) {
1162                 if (!adev->ip_blocks[i].status.valid)
1163                         continue;
1164                 if (adev->ip_blocks[i].version->type == block_type)
1165                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1166         }
1167         return true;
1168
1169 }
1170
1171 /**
1172  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1173  *
1174  * @adev: amdgpu_device pointer
1175  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1176  *
1177  * Returns a pointer to the hardware IP block structure
1178  * if it exists for the asic, otherwise NULL.
1179  */
1180 struct amdgpu_ip_block *
1181 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1182                               enum amd_ip_block_type type)
1183 {
1184         int i;
1185
1186         for (i = 0; i < adev->num_ip_blocks; i++)
1187                 if (adev->ip_blocks[i].version->type == type)
1188                         return &adev->ip_blocks[i];
1189
1190         return NULL;
1191 }
1192
1193 /**
1194  * amdgpu_device_ip_block_version_cmp
1195  *
1196  * @adev: amdgpu_device pointer
1197  * @type: enum amd_ip_block_type
1198  * @major: major version
1199  * @minor: minor version
1200  *
1201  * return 0 if equal or greater
1202  * return 1 if smaller or the ip_block doesn't exist
1203  */
1204 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1205                                        enum amd_ip_block_type type,
1206                                        u32 major, u32 minor)
1207 {
1208         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1209
1210         if (ip_block && ((ip_block->version->major > major) ||
1211                         ((ip_block->version->major == major) &&
1212                         (ip_block->version->minor >= minor))))
1213                 return 0;
1214
1215         return 1;
1216 }
1217
1218 /**
1219  * amdgpu_device_ip_block_add
1220  *
1221  * @adev: amdgpu_device pointer
1222  * @ip_block_version: pointer to the IP to add
1223  *
1224  * Adds the IP block driver information to the collection of IPs
1225  * on the asic.
1226  */
1227 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1228                                const struct amdgpu_ip_block_version *ip_block_version)
1229 {
1230         if (!ip_block_version)
1231                 return -EINVAL;
1232
1233         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1234                   ip_block_version->funcs->name);
1235
1236         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1237
1238         return 0;
1239 }
1240
1241 /**
1242  * amdgpu_device_enable_virtual_display - enable virtual display feature
1243  *
1244  * @adev: amdgpu_device pointer
1245  *
1246  * Enabled the virtual display feature if the user has enabled it via
1247  * the module parameter virtual_display.  This feature provides a virtual
1248  * display hardware on headless boards or in virtualized environments.
1249  * This function parses and validates the configuration string specified by
1250  * the user and configues the virtual display configuration (number of
1251  * virtual connectors, crtcs, etc.) specified.
1252  */
1253 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1254 {
1255         adev->enable_virtual_display = false;
1256
1257         if (amdgpu_virtual_display) {
1258                 struct drm_device *ddev = adev->ddev;
1259                 const char *pci_address_name = pci_name(ddev->pdev);
1260                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1261
1262                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1263                 pciaddstr_tmp = pciaddstr;
1264                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1265                         pciaddname = strsep(&pciaddname_tmp, ",");
1266                         if (!strcmp("all", pciaddname)
1267                             || !strcmp(pci_address_name, pciaddname)) {
1268                                 long num_crtc;
1269                                 int res = -1;
1270
1271                                 adev->enable_virtual_display = true;
1272
1273                                 if (pciaddname_tmp)
1274                                         res = kstrtol(pciaddname_tmp, 10,
1275                                                       &num_crtc);
1276
1277                                 if (!res) {
1278                                         if (num_crtc < 1)
1279                                                 num_crtc = 1;
1280                                         if (num_crtc > 6)
1281                                                 num_crtc = 6;
1282                                         adev->mode_info.num_crtc = num_crtc;
1283                                 } else {
1284                                         adev->mode_info.num_crtc = 1;
1285                                 }
1286                                 break;
1287                         }
1288                 }
1289
1290                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1291                          amdgpu_virtual_display, pci_address_name,
1292                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1293
1294                 kfree(pciaddstr);
1295         }
1296 }
1297
1298 /**
1299  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1300  *
1301  * @adev: amdgpu_device pointer
1302  *
1303  * Parses the asic configuration parameters specified in the gpu info
1304  * firmware and makes them availale to the driver for use in configuring
1305  * the asic.
1306  * Returns 0 on success, -EINVAL on failure.
1307  */
1308 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1309 {
1310         const char *chip_name;
1311         char fw_name[30];
1312         int err;
1313         const struct gpu_info_firmware_header_v1_0 *hdr;
1314
1315         adev->firmware.gpu_info_fw = NULL;
1316
1317         switch (adev->asic_type) {
1318         case CHIP_TOPAZ:
1319         case CHIP_TONGA:
1320         case CHIP_FIJI:
1321         case CHIP_POLARIS11:
1322         case CHIP_POLARIS10:
1323         case CHIP_POLARIS12:
1324         case CHIP_CARRIZO:
1325         case CHIP_STONEY:
1326 #ifdef CONFIG_DRM_AMDGPU_SI
1327         case CHIP_VERDE:
1328         case CHIP_TAHITI:
1329         case CHIP_PITCAIRN:
1330         case CHIP_OLAND:
1331         case CHIP_HAINAN:
1332 #endif
1333 #ifdef CONFIG_DRM_AMDGPU_CIK
1334         case CHIP_BONAIRE:
1335         case CHIP_HAWAII:
1336         case CHIP_KAVERI:
1337         case CHIP_KABINI:
1338         case CHIP_MULLINS:
1339 #endif
1340         default:
1341                 return 0;
1342         case CHIP_VEGA10:
1343                 chip_name = "vega10";
1344                 break;
1345         case CHIP_RAVEN:
1346                 chip_name = "raven";
1347                 break;
1348         }
1349
1350         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1351         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1352         if (err) {
1353                 dev_err(adev->dev,
1354                         "Failed to load gpu_info firmware \"%s\"\n",
1355                         fw_name);
1356                 goto out;
1357         }
1358         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1359         if (err) {
1360                 dev_err(adev->dev,
1361                         "Failed to validate gpu_info firmware \"%s\"\n",
1362                         fw_name);
1363                 goto out;
1364         }
1365
1366         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1367         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1368
1369         switch (hdr->version_major) {
1370         case 1:
1371         {
1372                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1373                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1374                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1375
1376                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1377                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1378                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1379                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1380                 adev->gfx.config.max_texture_channel_caches =
1381                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
1382                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1383                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1384                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1385                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1386                 adev->gfx.config.double_offchip_lds_buf =
1387                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1388                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1389                 adev->gfx.cu_info.max_waves_per_simd =
1390                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1391                 adev->gfx.cu_info.max_scratch_slots_per_cu =
1392                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1393                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1394                 break;
1395         }
1396         default:
1397                 dev_err(adev->dev,
1398                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1399                 err = -EINVAL;
1400                 goto out;
1401         }
1402 out:
1403         return err;
1404 }
1405
1406 /**
1407  * amdgpu_device_ip_early_init - run early init for hardware IPs
1408  *
1409  * @adev: amdgpu_device pointer
1410  *
1411  * Early initialization pass for hardware IPs.  The hardware IPs that make
1412  * up each asic are discovered each IP's early_init callback is run.  This
1413  * is the first stage in initializing the asic.
1414  * Returns 0 on success, negative error code on failure.
1415  */
1416 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1417 {
1418         int i, r;
1419
1420         amdgpu_device_enable_virtual_display(adev);
1421
1422         switch (adev->asic_type) {
1423         case CHIP_TOPAZ:
1424         case CHIP_TONGA:
1425         case CHIP_FIJI:
1426         case CHIP_POLARIS11:
1427         case CHIP_POLARIS10:
1428         case CHIP_POLARIS12:
1429         case CHIP_CARRIZO:
1430         case CHIP_STONEY:
1431                 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1432                         adev->family = AMDGPU_FAMILY_CZ;
1433                 else
1434                         adev->family = AMDGPU_FAMILY_VI;
1435
1436                 r = vi_set_ip_blocks(adev);
1437                 if (r)
1438                         return r;
1439                 break;
1440 #ifdef CONFIG_DRM_AMDGPU_SI
1441         case CHIP_VERDE:
1442         case CHIP_TAHITI:
1443         case CHIP_PITCAIRN:
1444         case CHIP_OLAND:
1445         case CHIP_HAINAN:
1446                 adev->family = AMDGPU_FAMILY_SI;
1447                 r = si_set_ip_blocks(adev);
1448                 if (r)
1449                         return r;
1450                 break;
1451 #endif
1452 #ifdef CONFIG_DRM_AMDGPU_CIK
1453         case CHIP_BONAIRE:
1454         case CHIP_HAWAII:
1455         case CHIP_KAVERI:
1456         case CHIP_KABINI:
1457         case CHIP_MULLINS:
1458                 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1459                         adev->family = AMDGPU_FAMILY_CI;
1460                 else
1461                         adev->family = AMDGPU_FAMILY_KV;
1462
1463                 r = cik_set_ip_blocks(adev);
1464                 if (r)
1465                         return r;
1466                 break;
1467 #endif
1468         case  CHIP_VEGA10:
1469         case  CHIP_RAVEN:
1470                 if (adev->asic_type == CHIP_RAVEN)
1471                         adev->family = AMDGPU_FAMILY_RV;
1472                 else
1473                         adev->family = AMDGPU_FAMILY_AI;
1474
1475                 r = soc15_set_ip_blocks(adev);
1476                 if (r)
1477                         return r;
1478                 break;
1479         default:
1480                 /* FIXME: not supported yet */
1481                 return -EINVAL;
1482         }
1483
1484         r = amdgpu_device_parse_gpu_info_fw(adev);
1485         if (r)
1486                 return r;
1487
1488         amdgpu_amdkfd_device_probe(adev);
1489
1490         if (amdgpu_sriov_vf(adev)) {
1491                 r = amdgpu_virt_request_full_gpu(adev, true);
1492                 if (r)
1493                         return -EAGAIN;
1494         }
1495
1496         for (i = 0; i < adev->num_ip_blocks; i++) {
1497                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1498                         DRM_ERROR("disabled ip block: %d <%s>\n",
1499                                   i, adev->ip_blocks[i].version->funcs->name);
1500                         adev->ip_blocks[i].status.valid = false;
1501                 } else {
1502                         if (adev->ip_blocks[i].version->funcs->early_init) {
1503                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1504                                 if (r == -ENOENT) {
1505                                         adev->ip_blocks[i].status.valid = false;
1506                                 } else if (r) {
1507                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
1508                                                   adev->ip_blocks[i].version->funcs->name, r);
1509                                         return r;
1510                                 } else {
1511                                         adev->ip_blocks[i].status.valid = true;
1512                                 }
1513                         } else {
1514                                 adev->ip_blocks[i].status.valid = true;
1515                         }
1516                 }
1517         }
1518
1519         adev->cg_flags &= amdgpu_cg_mask;
1520         adev->pg_flags &= amdgpu_pg_mask;
1521
1522         return 0;
1523 }
1524
1525 /**
1526  * amdgpu_device_ip_init - run init for hardware IPs
1527  *
1528  * @adev: amdgpu_device pointer
1529  *
1530  * Main initialization pass for hardware IPs.  The list of all the hardware
1531  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
1532  * are run.  sw_init initializes the software state associated with each IP
1533  * and hw_init initializes the hardware associated with each IP.
1534  * Returns 0 on success, negative error code on failure.
1535  */
1536 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
1537 {
1538         int i, r;
1539
1540         for (i = 0; i < adev->num_ip_blocks; i++) {
1541                 if (!adev->ip_blocks[i].status.valid)
1542                         continue;
1543                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1544                 if (r) {
1545                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1546                                   adev->ip_blocks[i].version->funcs->name, r);
1547                         return r;
1548                 }
1549                 adev->ip_blocks[i].status.sw = true;
1550
1551                 /* need to do gmc hw init early so we can allocate gpu mem */
1552                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1553                         r = amdgpu_device_vram_scratch_init(adev);
1554                         if (r) {
1555                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1556                                 return r;
1557                         }
1558                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1559                         if (r) {
1560                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
1561                                 return r;
1562                         }
1563                         r = amdgpu_device_wb_init(adev);
1564                         if (r) {
1565                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
1566                                 return r;
1567                         }
1568                         adev->ip_blocks[i].status.hw = true;
1569
1570                         /* right after GMC hw init, we create CSA */
1571                         if (amdgpu_sriov_vf(adev)) {
1572                                 r = amdgpu_allocate_static_csa(adev);
1573                                 if (r) {
1574                                         DRM_ERROR("allocate CSA failed %d\n", r);
1575                                         return r;
1576                                 }
1577                         }
1578                 }
1579         }
1580
1581         for (i = 0; i < adev->num_ip_blocks; i++) {
1582                 if (!adev->ip_blocks[i].status.sw)
1583                         continue;
1584                 if (adev->ip_blocks[i].status.hw)
1585                         continue;
1586                 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1587                 if (r) {
1588                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1589                                   adev->ip_blocks[i].version->funcs->name, r);
1590                         return r;
1591                 }
1592                 adev->ip_blocks[i].status.hw = true;
1593         }
1594
1595         amdgpu_amdkfd_device_init(adev);
1596
1597         if (amdgpu_sriov_vf(adev))
1598                 amdgpu_virt_release_full_gpu(adev, true);
1599
1600         return 0;
1601 }
1602
1603 /**
1604  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
1605  *
1606  * @adev: amdgpu_device pointer
1607  *
1608  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
1609  * this function before a GPU reset.  If the value is retained after a
1610  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
1611  */
1612 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
1613 {
1614         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1615 }
1616
1617 /**
1618  * amdgpu_device_check_vram_lost - check if vram is valid
1619  *
1620  * @adev: amdgpu_device pointer
1621  *
1622  * Checks the reset magic value written to the gart pointer in VRAM.
1623  * The driver calls this after a GPU reset to see if the contents of
1624  * VRAM is lost or now.
1625  * returns true if vram is lost, false if not.
1626  */
1627 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
1628 {
1629         return !!memcmp(adev->gart.ptr, adev->reset_magic,
1630                         AMDGPU_RESET_MAGIC_NUM);
1631 }
1632
1633 /**
1634  * amdgpu_device_ip_late_set_cg_state - late init for clockgating
1635  *
1636  * @adev: amdgpu_device pointer
1637  *
1638  * Late initialization pass enabling clockgating for hardware IPs.
1639  * The list of all the hardware IPs that make up the asic is walked and the
1640  * set_clockgating_state callbacks are run.  This stage is run late
1641  * in the init process.
1642  * Returns 0 on success, negative error code on failure.
1643  */
1644 static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev)
1645 {
1646         int i = 0, r;
1647
1648         if (amdgpu_emu_mode == 1)
1649                 return 0;
1650
1651         for (i = 0; i < adev->num_ip_blocks; i++) {
1652                 if (!adev->ip_blocks[i].status.valid)
1653                         continue;
1654                 /* skip CG for VCE/UVD, it's handled specially */
1655                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1656                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1657                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1658                         /* enable clockgating to save power */
1659                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1660                                                                                      AMD_CG_STATE_GATE);
1661                         if (r) {
1662                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1663                                           adev->ip_blocks[i].version->funcs->name, r);
1664                                 return r;
1665                         }
1666                 }
1667         }
1668         return 0;
1669 }
1670
1671 /**
1672  * amdgpu_device_ip_late_init - run late init for hardware IPs
1673  *
1674  * @adev: amdgpu_device pointer
1675  *
1676  * Late initialization pass for hardware IPs.  The list of all the hardware
1677  * IPs that make up the asic is walked and the late_init callbacks are run.
1678  * late_init covers any special initialization that an IP requires
1679  * after all of the have been initialized or something that needs to happen
1680  * late in the init process.
1681  * Returns 0 on success, negative error code on failure.
1682  */
1683 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
1684 {
1685         int i = 0, r;
1686
1687         for (i = 0; i < adev->num_ip_blocks; i++) {
1688                 if (!adev->ip_blocks[i].status.valid)
1689                         continue;
1690                 if (adev->ip_blocks[i].version->funcs->late_init) {
1691                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1692                         if (r) {
1693                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1694                                           adev->ip_blocks[i].version->funcs->name, r);
1695                                 return r;
1696                         }
1697                         adev->ip_blocks[i].status.late_initialized = true;
1698                 }
1699         }
1700
1701         mod_delayed_work(system_wq, &adev->late_init_work,
1702                         msecs_to_jiffies(AMDGPU_RESUME_MS));
1703
1704         amdgpu_device_fill_reset_magic(adev);
1705
1706         return 0;
1707 }
1708
1709 /**
1710  * amdgpu_device_ip_fini - run fini for hardware IPs
1711  *
1712  * @adev: amdgpu_device pointer
1713  *
1714  * Main teardown pass for hardware IPs.  The list of all the hardware
1715  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
1716  * are run.  hw_fini tears down the hardware associated with each IP
1717  * and sw_fini tears down any software state associated with each IP.
1718  * Returns 0 on success, negative error code on failure.
1719  */
1720 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
1721 {
1722         int i, r;
1723
1724         amdgpu_amdkfd_device_fini(adev);
1725         /* need to disable SMC first */
1726         for (i = 0; i < adev->num_ip_blocks; i++) {
1727                 if (!adev->ip_blocks[i].status.hw)
1728                         continue;
1729                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC &&
1730                         adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1731                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1732                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1733                                                                                      AMD_CG_STATE_UNGATE);
1734                         if (r) {
1735                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1736                                           adev->ip_blocks[i].version->funcs->name, r);
1737                                 return r;
1738                         }
1739                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1740                         /* XXX handle errors */
1741                         if (r) {
1742                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1743                                           adev->ip_blocks[i].version->funcs->name, r);
1744                         }
1745                         adev->ip_blocks[i].status.hw = false;
1746                         break;
1747                 }
1748         }
1749
1750         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1751                 if (!adev->ip_blocks[i].status.hw)
1752                         continue;
1753
1754                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1755                         adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1756                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1757                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1758                                                                                      AMD_CG_STATE_UNGATE);
1759                         if (r) {
1760                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1761                                           adev->ip_blocks[i].version->funcs->name, r);
1762                                 return r;
1763                         }
1764                 }
1765
1766                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1767                 /* XXX handle errors */
1768                 if (r) {
1769                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1770                                   adev->ip_blocks[i].version->funcs->name, r);
1771                 }
1772
1773                 adev->ip_blocks[i].status.hw = false;
1774         }
1775
1776
1777         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1778                 if (!adev->ip_blocks[i].status.sw)
1779                         continue;
1780
1781                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1782                         amdgpu_free_static_csa(adev);
1783                         amdgpu_device_wb_fini(adev);
1784                         amdgpu_device_vram_scratch_fini(adev);
1785                 }
1786
1787                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1788                 /* XXX handle errors */
1789                 if (r) {
1790                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1791                                   adev->ip_blocks[i].version->funcs->name, r);
1792                 }
1793                 adev->ip_blocks[i].status.sw = false;
1794                 adev->ip_blocks[i].status.valid = false;
1795         }
1796
1797         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1798                 if (!adev->ip_blocks[i].status.late_initialized)
1799                         continue;
1800                 if (adev->ip_blocks[i].version->funcs->late_fini)
1801                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1802                 adev->ip_blocks[i].status.late_initialized = false;
1803         }
1804
1805         if (amdgpu_sriov_vf(adev))
1806                 if (amdgpu_virt_release_full_gpu(adev, false))
1807                         DRM_ERROR("failed to release exclusive mode on fini\n");
1808
1809         return 0;
1810 }
1811
1812 /**
1813  * amdgpu_device_ip_late_init_func_handler - work handler for clockgating
1814  *
1815  * @work: work_struct
1816  *
1817  * Work handler for amdgpu_device_ip_late_set_cg_state.  We put the
1818  * clockgating setup into a worker thread to speed up driver init and
1819  * resume from suspend.
1820  */
1821 static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
1822 {
1823         struct amdgpu_device *adev =
1824                 container_of(work, struct amdgpu_device, late_init_work.work);
1825         amdgpu_device_ip_late_set_cg_state(adev);
1826 }
1827
1828 /**
1829  * amdgpu_device_ip_suspend - run suspend for hardware IPs
1830  *
1831  * @adev: amdgpu_device pointer
1832  *
1833  * Main suspend function for hardware IPs.  The list of all the hardware
1834  * IPs that make up the asic is walked, clockgating is disabled and the
1835  * suspend callbacks are run.  suspend puts the hardware and software state
1836  * in each IP into a state suitable for suspend.
1837  * Returns 0 on success, negative error code on failure.
1838  */
1839 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
1840 {
1841         int i, r;
1842
1843         if (amdgpu_sriov_vf(adev))
1844                 amdgpu_virt_request_full_gpu(adev, false);
1845
1846         /* ungate SMC block first */
1847         r = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1848                                                    AMD_CG_STATE_UNGATE);
1849         if (r) {
1850                 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n", r);
1851         }
1852
1853         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1854                 if (!adev->ip_blocks[i].status.valid)
1855                         continue;
1856                 /* ungate blocks so that suspend can properly shut them down */
1857                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_SMC &&
1858                         adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1859                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1860                                                                                      AMD_CG_STATE_UNGATE);
1861                         if (r) {
1862                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1863                                           adev->ip_blocks[i].version->funcs->name, r);
1864                         }
1865                 }
1866                 /* XXX handle errors */
1867                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1868                 /* XXX handle errors */
1869                 if (r) {
1870                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
1871                                   adev->ip_blocks[i].version->funcs->name, r);
1872                 }
1873         }
1874
1875         if (amdgpu_sriov_vf(adev))
1876                 amdgpu_virt_release_full_gpu(adev, false);
1877
1878         return 0;
1879 }
1880
1881 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
1882 {
1883         int i, r;
1884
1885         static enum amd_ip_block_type ip_order[] = {
1886                 AMD_IP_BLOCK_TYPE_GMC,
1887                 AMD_IP_BLOCK_TYPE_COMMON,
1888                 AMD_IP_BLOCK_TYPE_IH,
1889         };
1890
1891         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1892                 int j;
1893                 struct amdgpu_ip_block *block;
1894
1895                 for (j = 0; j < adev->num_ip_blocks; j++) {
1896                         block = &adev->ip_blocks[j];
1897
1898                         if (block->version->type != ip_order[i] ||
1899                                 !block->status.valid)
1900                                 continue;
1901
1902                         r = block->version->funcs->hw_init(adev);
1903                         DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1904                         if (r)
1905                                 return r;
1906                 }
1907         }
1908
1909         return 0;
1910 }
1911
1912 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
1913 {
1914         int i, r;
1915
1916         static enum amd_ip_block_type ip_order[] = {
1917                 AMD_IP_BLOCK_TYPE_SMC,
1918                 AMD_IP_BLOCK_TYPE_PSP,
1919                 AMD_IP_BLOCK_TYPE_DCE,
1920                 AMD_IP_BLOCK_TYPE_GFX,
1921                 AMD_IP_BLOCK_TYPE_SDMA,
1922                 AMD_IP_BLOCK_TYPE_UVD,
1923                 AMD_IP_BLOCK_TYPE_VCE
1924         };
1925
1926         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1927                 int j;
1928                 struct amdgpu_ip_block *block;
1929
1930                 for (j = 0; j < adev->num_ip_blocks; j++) {
1931                         block = &adev->ip_blocks[j];
1932
1933                         if (block->version->type != ip_order[i] ||
1934                                 !block->status.valid)
1935                                 continue;
1936
1937                         r = block->version->funcs->hw_init(adev);
1938                         DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1939                         if (r)
1940                                 return r;
1941                 }
1942         }
1943
1944         return 0;
1945 }
1946
1947 /**
1948  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
1949  *
1950  * @adev: amdgpu_device pointer
1951  *
1952  * First resume function for hardware IPs.  The list of all the hardware
1953  * IPs that make up the asic is walked and the resume callbacks are run for
1954  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
1955  * after a suspend and updates the software state as necessary.  This
1956  * function is also used for restoring the GPU after a GPU reset.
1957  * Returns 0 on success, negative error code on failure.
1958  */
1959 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
1960 {
1961         int i, r;
1962
1963         for (i = 0; i < adev->num_ip_blocks; i++) {
1964                 if (!adev->ip_blocks[i].status.valid)
1965                         continue;
1966                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1967                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1968                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
1969                         r = adev->ip_blocks[i].version->funcs->resume(adev);
1970                         if (r) {
1971                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
1972                                           adev->ip_blocks[i].version->funcs->name, r);
1973                                 return r;
1974                         }
1975                 }
1976         }
1977
1978         return 0;
1979 }
1980
1981 /**
1982  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
1983  *
1984  * @adev: amdgpu_device pointer
1985  *
1986  * First resume function for hardware IPs.  The list of all the hardware
1987  * IPs that make up the asic is walked and the resume callbacks are run for
1988  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
1989  * functional state after a suspend and updates the software state as
1990  * necessary.  This function is also used for restoring the GPU after a GPU
1991  * reset.
1992  * Returns 0 on success, negative error code on failure.
1993  */
1994 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
1995 {
1996         int i, r;
1997
1998         for (i = 0; i < adev->num_ip_blocks; i++) {
1999                 if (!adev->ip_blocks[i].status.valid)
2000                         continue;
2001                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2002                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2003                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)
2004                         continue;
2005                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2006                 if (r) {
2007                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2008                                   adev->ip_blocks[i].version->funcs->name, r);
2009                         return r;
2010                 }
2011         }
2012
2013         return 0;
2014 }
2015
2016 /**
2017  * amdgpu_device_ip_resume - run resume for hardware IPs
2018  *
2019  * @adev: amdgpu_device pointer
2020  *
2021  * Main resume function for hardware IPs.  The hardware IPs
2022  * are split into two resume functions because they are
2023  * are also used in in recovering from a GPU reset and some additional
2024  * steps need to be take between them.  In this case (S3/S4) they are
2025  * run sequentially.
2026  * Returns 0 on success, negative error code on failure.
2027  */
2028 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
2029 {
2030         int r;
2031
2032         r = amdgpu_device_ip_resume_phase1(adev);
2033         if (r)
2034                 return r;
2035         r = amdgpu_device_ip_resume_phase2(adev);
2036
2037         return r;
2038 }
2039
2040 /**
2041  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
2042  *
2043  * @adev: amdgpu_device pointer
2044  *
2045  * Query the VBIOS data tables to determine if the board supports SR-IOV.
2046  */
2047 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2048 {
2049         if (amdgpu_sriov_vf(adev)) {
2050                 if (adev->is_atom_fw) {
2051                         if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2052                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2053                 } else {
2054                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2055                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2056                 }
2057
2058                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
2059                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2060         }
2061 }
2062
2063 /**
2064  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
2065  *
2066  * @asic_type: AMD asic type
2067  *
2068  * Check if there is DC (new modesetting infrastructre) support for an asic.
2069  * returns true if DC has support, false if not.
2070  */
2071 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
2072 {
2073         switch (asic_type) {
2074 #if defined(CONFIG_DRM_AMD_DC)
2075         case CHIP_BONAIRE:
2076         case CHIP_HAWAII:
2077         case CHIP_KAVERI:
2078         case CHIP_KABINI:
2079         case CHIP_MULLINS:
2080         case CHIP_CARRIZO:
2081         case CHIP_STONEY:
2082         case CHIP_POLARIS11:
2083         case CHIP_POLARIS10:
2084         case CHIP_POLARIS12:
2085         case CHIP_TONGA:
2086         case CHIP_FIJI:
2087 #if defined(CONFIG_DRM_AMD_DC_PRE_VEGA)
2088                 return amdgpu_dc != 0;
2089 #endif
2090         case CHIP_VEGA10:
2091 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
2092         case CHIP_RAVEN:
2093 #endif
2094                 return amdgpu_dc != 0;
2095 #endif
2096         default:
2097                 return false;
2098         }
2099 }
2100
2101 /**
2102  * amdgpu_device_has_dc_support - check if dc is supported
2103  *
2104  * @adev: amdgpu_device_pointer
2105  *
2106  * Returns true for supported, false for not supported
2107  */
2108 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
2109 {
2110         if (amdgpu_sriov_vf(adev))
2111                 return false;
2112
2113         return amdgpu_device_asic_has_dc_support(adev->asic_type);
2114 }
2115
2116 /**
2117  * amdgpu_device_init - initialize the driver
2118  *
2119  * @adev: amdgpu_device pointer
2120  * @pdev: drm dev pointer
2121  * @pdev: pci dev pointer
2122  * @flags: driver flags
2123  *
2124  * Initializes the driver info and hw (all asics).
2125  * Returns 0 for success or an error on failure.
2126  * Called at driver startup.
2127  */
2128 int amdgpu_device_init(struct amdgpu_device *adev,
2129                        struct drm_device *ddev,
2130                        struct pci_dev *pdev,
2131                        uint32_t flags)
2132 {
2133         int r, i;
2134         bool runtime = false;
2135         u32 max_MBps;
2136
2137         adev->shutdown = false;
2138         adev->dev = &pdev->dev;
2139         adev->ddev = ddev;
2140         adev->pdev = pdev;
2141         adev->flags = flags;
2142         adev->asic_type = flags & AMD_ASIC_MASK;
2143         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2144         if (amdgpu_emu_mode == 1)
2145                 adev->usec_timeout *= 2;
2146         adev->gmc.gart_size = 512 * 1024 * 1024;
2147         adev->accel_working = false;
2148         adev->num_rings = 0;
2149         adev->mman.buffer_funcs = NULL;
2150         adev->mman.buffer_funcs_ring = NULL;
2151         adev->vm_manager.vm_pte_funcs = NULL;
2152         adev->vm_manager.vm_pte_num_rings = 0;
2153         adev->gmc.gmc_funcs = NULL;
2154         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2155         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2156
2157         adev->smc_rreg = &amdgpu_invalid_rreg;
2158         adev->smc_wreg = &amdgpu_invalid_wreg;
2159         adev->pcie_rreg = &amdgpu_invalid_rreg;
2160         adev->pcie_wreg = &amdgpu_invalid_wreg;
2161         adev->pciep_rreg = &amdgpu_invalid_rreg;
2162         adev->pciep_wreg = &amdgpu_invalid_wreg;
2163         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2164         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2165         adev->didt_rreg = &amdgpu_invalid_rreg;
2166         adev->didt_wreg = &amdgpu_invalid_wreg;
2167         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2168         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
2169         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2170         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2171
2172         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2173                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2174                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
2175
2176         /* mutex initialization are all done here so we
2177          * can recall function without having locking issues */
2178         atomic_set(&adev->irq.ih.lock, 0);
2179         mutex_init(&adev->firmware.mutex);
2180         mutex_init(&adev->pm.mutex);
2181         mutex_init(&adev->gfx.gpu_clock_mutex);
2182         mutex_init(&adev->srbm_mutex);
2183         mutex_init(&adev->gfx.pipe_reserve_mutex);
2184         mutex_init(&adev->grbm_idx_mutex);
2185         mutex_init(&adev->mn_lock);
2186         mutex_init(&adev->virt.vf_errors.lock);
2187         hash_init(adev->mn_hash);
2188         mutex_init(&adev->lock_reset);
2189
2190         amdgpu_device_check_arguments(adev);
2191
2192         spin_lock_init(&adev->mmio_idx_lock);
2193         spin_lock_init(&adev->smc_idx_lock);
2194         spin_lock_init(&adev->pcie_idx_lock);
2195         spin_lock_init(&adev->uvd_ctx_idx_lock);
2196         spin_lock_init(&adev->didt_idx_lock);
2197         spin_lock_init(&adev->gc_cac_idx_lock);
2198         spin_lock_init(&adev->se_cac_idx_lock);
2199         spin_lock_init(&adev->audio_endpt_idx_lock);
2200         spin_lock_init(&adev->mm_stats.lock);
2201
2202         INIT_LIST_HEAD(&adev->shadow_list);
2203         mutex_init(&adev->shadow_list_lock);
2204
2205         INIT_LIST_HEAD(&adev->ring_lru_list);
2206         spin_lock_init(&adev->ring_lru_list_lock);
2207
2208         INIT_DELAYED_WORK(&adev->late_init_work,
2209                           amdgpu_device_ip_late_init_func_handler);
2210
2211         /* Registers mapping */
2212         /* TODO: block userspace mapping of io register */
2213         if (adev->asic_type >= CHIP_BONAIRE) {
2214                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2215                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2216         } else {
2217                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2218                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2219         }
2220
2221         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2222         if (adev->rmmio == NULL) {
2223                 return -ENOMEM;
2224         }
2225         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2226         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2227
2228         /* doorbell bar mapping */
2229         amdgpu_device_doorbell_init(adev);
2230
2231         /* io port mapping */
2232         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2233                 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2234                         adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2235                         adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2236                         break;
2237                 }
2238         }
2239         if (adev->rio_mem == NULL)
2240                 DRM_INFO("PCI I/O BAR is not found.\n");
2241
2242         amdgpu_device_get_pcie_info(adev);
2243
2244         /* early init functions */
2245         r = amdgpu_device_ip_early_init(adev);
2246         if (r)
2247                 return r;
2248
2249         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2250         /* this will fail for cards that aren't VGA class devices, just
2251          * ignore it */
2252         vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
2253
2254         if (amdgpu_device_is_px(ddev))
2255                 runtime = true;
2256         if (!pci_is_thunderbolt_attached(adev->pdev))
2257                 vga_switcheroo_register_client(adev->pdev,
2258                                                &amdgpu_switcheroo_ops, runtime);
2259         if (runtime)
2260                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2261
2262         if (amdgpu_emu_mode == 1) {
2263                 /* post the asic on emulation mode */
2264                 emu_soc_asic_init(adev);
2265                 goto fence_driver_init;
2266         }
2267
2268         /* Read BIOS */
2269         if (!amdgpu_get_bios(adev)) {
2270                 r = -EINVAL;
2271                 goto failed;
2272         }
2273
2274         r = amdgpu_atombios_init(adev);
2275         if (r) {
2276                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2277                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2278                 goto failed;
2279         }
2280
2281         /* detect if we are with an SRIOV vbios */
2282         amdgpu_device_detect_sriov_bios(adev);
2283
2284         /* Post card if necessary */
2285         if (amdgpu_device_need_post(adev)) {
2286                 if (!adev->bios) {
2287                         dev_err(adev->dev, "no vBIOS found\n");
2288                         r = -EINVAL;
2289                         goto failed;
2290                 }
2291                 DRM_INFO("GPU posting now...\n");
2292                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2293                 if (r) {
2294                         dev_err(adev->dev, "gpu post error!\n");
2295                         goto failed;
2296                 }
2297         }
2298
2299         if (adev->is_atom_fw) {
2300                 /* Initialize clocks */
2301                 r = amdgpu_atomfirmware_get_clock_info(adev);
2302                 if (r) {
2303                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
2304                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2305                         goto failed;
2306                 }
2307         } else {
2308                 /* Initialize clocks */
2309                 r = amdgpu_atombios_get_clock_info(adev);
2310                 if (r) {
2311                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2312                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2313                         goto failed;
2314                 }
2315                 /* init i2c buses */
2316                 if (!amdgpu_device_has_dc_support(adev))
2317                         amdgpu_atombios_i2c_init(adev);
2318         }
2319
2320 fence_driver_init:
2321         /* Fence driver */
2322         r = amdgpu_fence_driver_init(adev);
2323         if (r) {
2324                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2325                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2326                 goto failed;
2327         }
2328
2329         /* init the mode config */
2330         drm_mode_config_init(adev->ddev);
2331
2332         r = amdgpu_device_ip_init(adev);
2333         if (r) {
2334                 /* failed in exclusive mode due to timeout */
2335                 if (amdgpu_sriov_vf(adev) &&
2336                     !amdgpu_sriov_runtime(adev) &&
2337                     amdgpu_virt_mmio_blocked(adev) &&
2338                     !amdgpu_virt_wait_reset(adev)) {
2339                         dev_err(adev->dev, "VF exclusive mode timeout\n");
2340                         /* Don't send request since VF is inactive. */
2341                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
2342                         adev->virt.ops = NULL;
2343                         r = -EAGAIN;
2344                         goto failed;
2345                 }
2346                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
2347                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2348                 amdgpu_device_ip_fini(adev);
2349                 goto failed;
2350         }
2351
2352         adev->accel_working = true;
2353
2354         amdgpu_vm_check_compute_bug(adev);
2355
2356         /* Initialize the buffer migration limit. */
2357         if (amdgpu_moverate >= 0)
2358                 max_MBps = amdgpu_moverate;
2359         else
2360                 max_MBps = 8; /* Allow 8 MB/s. */
2361         /* Get a log2 for easy divisions. */
2362         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2363
2364         r = amdgpu_ib_pool_init(adev);
2365         if (r) {
2366                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2367                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2368                 goto failed;
2369         }
2370
2371         r = amdgpu_ib_ring_tests(adev);
2372         if (r)
2373                 DRM_ERROR("ib ring test failed (%d).\n", r);
2374
2375         if (amdgpu_sriov_vf(adev))
2376                 amdgpu_virt_init_data_exchange(adev);
2377
2378         amdgpu_fbdev_init(adev);
2379
2380         r = amdgpu_pm_sysfs_init(adev);
2381         if (r)
2382                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2383
2384         r = amdgpu_debugfs_gem_init(adev);
2385         if (r)
2386                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2387
2388         r = amdgpu_debugfs_regs_init(adev);
2389         if (r)
2390                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2391
2392         r = amdgpu_debugfs_firmware_init(adev);
2393         if (r)
2394                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2395
2396         r = amdgpu_debugfs_init(adev);
2397         if (r)
2398                 DRM_ERROR("Creating debugfs files failed (%d).\n", r);
2399
2400         if ((amdgpu_testing & 1)) {
2401                 if (adev->accel_working)
2402                         amdgpu_test_moves(adev);
2403                 else
2404                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2405         }
2406         if (amdgpu_benchmarking) {
2407                 if (adev->accel_working)
2408                         amdgpu_benchmark(adev, amdgpu_benchmarking);
2409                 else
2410                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2411         }
2412
2413         /* enable clockgating, etc. after ib tests, etc. since some blocks require
2414          * explicit gating rather than handling it automatically.
2415          */
2416         r = amdgpu_device_ip_late_init(adev);
2417         if (r) {
2418                 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
2419                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2420                 goto failed;
2421         }
2422
2423         return 0;
2424
2425 failed:
2426         amdgpu_vf_error_trans_all(adev);
2427         if (runtime)
2428                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2429
2430         return r;
2431 }
2432
2433 /**
2434  * amdgpu_device_fini - tear down the driver
2435  *
2436  * @adev: amdgpu_device pointer
2437  *
2438  * Tear down the driver info (all asics).
2439  * Called at driver shutdown.
2440  */
2441 void amdgpu_device_fini(struct amdgpu_device *adev)
2442 {
2443         int r;
2444
2445         DRM_INFO("amdgpu: finishing device.\n");
2446         adev->shutdown = true;
2447         /* disable all interrupts */
2448         amdgpu_irq_disable_all(adev);
2449         if (adev->mode_info.mode_config_initialized){
2450                 if (!amdgpu_device_has_dc_support(adev))
2451                         drm_crtc_force_disable_all(adev->ddev);
2452                 else
2453                         drm_atomic_helper_shutdown(adev->ddev);
2454         }
2455         amdgpu_ib_pool_fini(adev);
2456         amdgpu_fence_driver_fini(adev);
2457         amdgpu_pm_sysfs_fini(adev);
2458         amdgpu_fbdev_fini(adev);
2459         r = amdgpu_device_ip_fini(adev);
2460         if (adev->firmware.gpu_info_fw) {
2461                 release_firmware(adev->firmware.gpu_info_fw);
2462                 adev->firmware.gpu_info_fw = NULL;
2463         }
2464         adev->accel_working = false;
2465         cancel_delayed_work_sync(&adev->late_init_work);
2466         /* free i2c buses */
2467         if (!amdgpu_device_has_dc_support(adev))
2468                 amdgpu_i2c_fini(adev);
2469
2470         if (amdgpu_emu_mode != 1)
2471                 amdgpu_atombios_fini(adev);
2472
2473         kfree(adev->bios);
2474         adev->bios = NULL;
2475         if (!pci_is_thunderbolt_attached(adev->pdev))
2476                 vga_switcheroo_unregister_client(adev->pdev);
2477         if (adev->flags & AMD_IS_PX)
2478                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2479         vga_client_register(adev->pdev, NULL, NULL, NULL);
2480         if (adev->rio_mem)
2481                 pci_iounmap(adev->pdev, adev->rio_mem);
2482         adev->rio_mem = NULL;
2483         iounmap(adev->rmmio);
2484         adev->rmmio = NULL;
2485         amdgpu_device_doorbell_fini(adev);
2486         amdgpu_debugfs_regs_cleanup(adev);
2487 }
2488
2489
2490 /*
2491  * Suspend & resume.
2492  */
2493 /**
2494  * amdgpu_device_suspend - initiate device suspend
2495  *
2496  * @pdev: drm dev pointer
2497  * @state: suspend state
2498  *
2499  * Puts the hw in the suspend state (all asics).
2500  * Returns 0 for success or an error on failure.
2501  * Called at driver suspend.
2502  */
2503 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2504 {
2505         struct amdgpu_device *adev;
2506         struct drm_crtc *crtc;
2507         struct drm_connector *connector;
2508         int r;
2509
2510         if (dev == NULL || dev->dev_private == NULL) {
2511                 return -ENODEV;
2512         }
2513
2514         adev = dev->dev_private;
2515
2516         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2517                 return 0;
2518
2519         drm_kms_helper_poll_disable(dev);
2520
2521         if (!amdgpu_device_has_dc_support(adev)) {
2522                 /* turn off display hw */
2523                 drm_modeset_lock_all(dev);
2524                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2525                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2526                 }
2527                 drm_modeset_unlock_all(dev);
2528         }
2529
2530         amdgpu_amdkfd_suspend(adev);
2531
2532         /* unpin the front buffers and cursors */
2533         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2534                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2535                 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
2536                 struct amdgpu_bo *robj;
2537
2538                 if (amdgpu_crtc->cursor_bo) {
2539                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2540                         r = amdgpu_bo_reserve(aobj, true);
2541                         if (r == 0) {
2542                                 amdgpu_bo_unpin(aobj);
2543                                 amdgpu_bo_unreserve(aobj);
2544                         }
2545                 }
2546
2547                 if (rfb == NULL || rfb->obj == NULL) {
2548                         continue;
2549                 }
2550                 robj = gem_to_amdgpu_bo(rfb->obj);
2551                 /* don't unpin kernel fb objects */
2552                 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2553                         r = amdgpu_bo_reserve(robj, true);
2554                         if (r == 0) {
2555                                 amdgpu_bo_unpin(robj);
2556                                 amdgpu_bo_unreserve(robj);
2557                         }
2558                 }
2559         }
2560         /* evict vram memory */
2561         amdgpu_bo_evict_vram(adev);
2562
2563         amdgpu_fence_driver_suspend(adev);
2564
2565         r = amdgpu_device_ip_suspend(adev);
2566
2567         /* evict remaining vram memory
2568          * This second call to evict vram is to evict the gart page table
2569          * using the CPU.
2570          */
2571         amdgpu_bo_evict_vram(adev);
2572
2573         pci_save_state(dev->pdev);
2574         if (suspend) {
2575                 /* Shut down the device */
2576                 pci_disable_device(dev->pdev);
2577                 pci_set_power_state(dev->pdev, PCI_D3hot);
2578         } else {
2579                 r = amdgpu_asic_reset(adev);
2580                 if (r)
2581                         DRM_ERROR("amdgpu asic reset failed\n");
2582         }
2583
2584         if (fbcon) {
2585                 console_lock();
2586                 amdgpu_fbdev_set_suspend(adev, 1);
2587                 console_unlock();
2588         }
2589         return 0;
2590 }
2591
2592 /**
2593  * amdgpu_device_resume - initiate device resume
2594  *
2595  * @pdev: drm dev pointer
2596  *
2597  * Bring the hw back to operating state (all asics).
2598  * Returns 0 for success or an error on failure.
2599  * Called at driver resume.
2600  */
2601 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2602 {
2603         struct drm_connector *connector;
2604         struct amdgpu_device *adev = dev->dev_private;
2605         struct drm_crtc *crtc;
2606         int r = 0;
2607
2608         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2609                 return 0;
2610
2611         if (fbcon)
2612                 console_lock();
2613
2614         if (resume) {
2615                 pci_set_power_state(dev->pdev, PCI_D0);
2616                 pci_restore_state(dev->pdev);
2617                 r = pci_enable_device(dev->pdev);
2618                 if (r)
2619                         goto unlock;
2620         }
2621
2622         /* post card */
2623         if (amdgpu_device_need_post(adev)) {
2624                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2625                 if (r)
2626                         DRM_ERROR("amdgpu asic init failed\n");
2627         }
2628
2629         r = amdgpu_device_ip_resume(adev);
2630         if (r) {
2631                 DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
2632                 goto unlock;
2633         }
2634         amdgpu_fence_driver_resume(adev);
2635
2636         if (resume) {
2637                 r = amdgpu_ib_ring_tests(adev);
2638                 if (r)
2639                         DRM_ERROR("ib ring test failed (%d).\n", r);
2640         }
2641
2642         r = amdgpu_device_ip_late_init(adev);
2643         if (r)
2644                 goto unlock;
2645
2646         /* pin cursors */
2647         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2648                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2649
2650                 if (amdgpu_crtc->cursor_bo) {
2651                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2652                         r = amdgpu_bo_reserve(aobj, true);
2653                         if (r == 0) {
2654                                 r = amdgpu_bo_pin(aobj,
2655                                                   AMDGPU_GEM_DOMAIN_VRAM,
2656                                                   &amdgpu_crtc->cursor_addr);
2657                                 if (r != 0)
2658                                         DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2659                                 amdgpu_bo_unreserve(aobj);
2660                         }
2661                 }
2662         }
2663         r = amdgpu_amdkfd_resume(adev);
2664         if (r)
2665                 return r;
2666
2667         /* blat the mode back in */
2668         if (fbcon) {
2669                 if (!amdgpu_device_has_dc_support(adev)) {
2670                         /* pre DCE11 */
2671                         drm_helper_resume_force_mode(dev);
2672
2673                         /* turn on display hw */
2674                         drm_modeset_lock_all(dev);
2675                         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2676                                 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2677                         }
2678                         drm_modeset_unlock_all(dev);
2679                 }
2680         }
2681
2682         drm_kms_helper_poll_enable(dev);
2683
2684         /*
2685          * Most of the connector probing functions try to acquire runtime pm
2686          * refs to ensure that the GPU is powered on when connector polling is
2687          * performed. Since we're calling this from a runtime PM callback,
2688          * trying to acquire rpm refs will cause us to deadlock.
2689          *
2690          * Since we're guaranteed to be holding the rpm lock, it's safe to
2691          * temporarily disable the rpm helpers so this doesn't deadlock us.
2692          */
2693 #ifdef CONFIG_PM
2694         dev->dev->power.disable_depth++;
2695 #endif
2696         if (!amdgpu_device_has_dc_support(adev))
2697                 drm_helper_hpd_irq_event(dev);
2698         else
2699                 drm_kms_helper_hotplug_event(dev);
2700 #ifdef CONFIG_PM
2701         dev->dev->power.disable_depth--;
2702 #endif
2703
2704         if (fbcon)
2705                 amdgpu_fbdev_set_suspend(adev, 0);
2706
2707 unlock:
2708         if (fbcon)
2709                 console_unlock();
2710
2711         return r;
2712 }
2713
2714 /**
2715  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
2716  *
2717  * @adev: amdgpu_device pointer
2718  *
2719  * The list of all the hardware IPs that make up the asic is walked and
2720  * the check_soft_reset callbacks are run.  check_soft_reset determines
2721  * if the asic is still hung or not.
2722  * Returns true if any of the IPs are still in a hung state, false if not.
2723  */
2724 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
2725 {
2726         int i;
2727         bool asic_hang = false;
2728
2729         if (amdgpu_sriov_vf(adev))
2730                 return true;
2731
2732         for (i = 0; i < adev->num_ip_blocks; i++) {
2733                 if (!adev->ip_blocks[i].status.valid)
2734                         continue;
2735                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2736                         adev->ip_blocks[i].status.hang =
2737                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2738                 if (adev->ip_blocks[i].status.hang) {
2739                         DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2740                         asic_hang = true;
2741                 }
2742         }
2743         return asic_hang;
2744 }
2745
2746 /**
2747  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
2748  *
2749  * @adev: amdgpu_device pointer
2750  *
2751  * The list of all the hardware IPs that make up the asic is walked and the
2752  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
2753  * handles any IP specific hardware or software state changes that are
2754  * necessary for a soft reset to succeed.
2755  * Returns 0 on success, negative error code on failure.
2756  */
2757 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
2758 {
2759         int i, r = 0;
2760
2761         for (i = 0; i < adev->num_ip_blocks; i++) {
2762                 if (!adev->ip_blocks[i].status.valid)
2763                         continue;
2764                 if (adev->ip_blocks[i].status.hang &&
2765                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2766                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2767                         if (r)
2768                                 return r;
2769                 }
2770         }
2771
2772         return 0;
2773 }
2774
2775 /**
2776  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
2777  *
2778  * @adev: amdgpu_device pointer
2779  *
2780  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
2781  * reset is necessary to recover.
2782  * Returns true if a full asic reset is required, false if not.
2783  */
2784 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
2785 {
2786         int i;
2787
2788         for (i = 0; i < adev->num_ip_blocks; i++) {
2789                 if (!adev->ip_blocks[i].status.valid)
2790                         continue;
2791                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2792                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2793                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2794                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
2795                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2796                         if (adev->ip_blocks[i].status.hang) {
2797                                 DRM_INFO("Some block need full reset!\n");
2798                                 return true;
2799                         }
2800                 }
2801         }
2802         return false;
2803 }
2804
2805 /**
2806  * amdgpu_device_ip_soft_reset - do a soft reset
2807  *
2808  * @adev: amdgpu_device pointer
2809  *
2810  * The list of all the hardware IPs that make up the asic is walked and the
2811  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
2812  * IP specific hardware or software state changes that are necessary to soft
2813  * reset the IP.
2814  * Returns 0 on success, negative error code on failure.
2815  */
2816 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
2817 {
2818         int i, r = 0;
2819
2820         for (i = 0; i < adev->num_ip_blocks; i++) {
2821                 if (!adev->ip_blocks[i].status.valid)
2822                         continue;
2823                 if (adev->ip_blocks[i].status.hang &&
2824                     adev->ip_blocks[i].version->funcs->soft_reset) {
2825                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2826                         if (r)
2827                                 return r;
2828                 }
2829         }
2830
2831         return 0;
2832 }
2833
2834 /**
2835  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
2836  *
2837  * @adev: amdgpu_device pointer
2838  *
2839  * The list of all the hardware IPs that make up the asic is walked and the
2840  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
2841  * handles any IP specific hardware or software state changes that are
2842  * necessary after the IP has been soft reset.
2843  * Returns 0 on success, negative error code on failure.
2844  */
2845 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
2846 {
2847         int i, r = 0;
2848
2849         for (i = 0; i < adev->num_ip_blocks; i++) {
2850                 if (!adev->ip_blocks[i].status.valid)
2851                         continue;
2852                 if (adev->ip_blocks[i].status.hang &&
2853                     adev->ip_blocks[i].version->funcs->post_soft_reset)
2854                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2855                 if (r)
2856                         return r;
2857         }
2858
2859         return 0;
2860 }
2861
2862 /**
2863  * amdgpu_device_recover_vram_from_shadow - restore shadowed VRAM buffers
2864  *
2865  * @adev: amdgpu_device pointer
2866  * @ring: amdgpu_ring for the engine handling the buffer operations
2867  * @bo: amdgpu_bo buffer whose shadow is being restored
2868  * @fence: dma_fence associated with the operation
2869  *
2870  * Restores the VRAM buffer contents from the shadow in GTT.  Used to
2871  * restore things like GPUVM page tables after a GPU reset where
2872  * the contents of VRAM might be lost.
2873  * Returns 0 on success, negative error code on failure.
2874  */
2875 static int amdgpu_device_recover_vram_from_shadow(struct amdgpu_device *adev,
2876                                                   struct amdgpu_ring *ring,
2877                                                   struct amdgpu_bo *bo,
2878                                                   struct dma_fence **fence)
2879 {
2880         uint32_t domain;
2881         int r;
2882
2883         if (!bo->shadow)
2884                 return 0;
2885
2886         r = amdgpu_bo_reserve(bo, true);
2887         if (r)
2888                 return r;
2889         domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2890         /* if bo has been evicted, then no need to recover */
2891         if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2892                 r = amdgpu_bo_validate(bo->shadow);
2893                 if (r) {
2894                         DRM_ERROR("bo validate failed!\n");
2895                         goto err;
2896                 }
2897
2898                 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2899                                                  NULL, fence, true);
2900                 if (r) {
2901                         DRM_ERROR("recover page table failed!\n");
2902                         goto err;
2903                 }
2904         }
2905 err:
2906         amdgpu_bo_unreserve(bo);
2907         return r;
2908 }
2909
2910 /**
2911  * amdgpu_device_handle_vram_lost - Handle the loss of VRAM contents
2912  *
2913  * @adev: amdgpu_device pointer
2914  *
2915  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
2916  * restore things like GPUVM page tables after a GPU reset where
2917  * the contents of VRAM might be lost.
2918  * Returns 0 on success, 1 on failure.
2919  */
2920 static int amdgpu_device_handle_vram_lost(struct amdgpu_device *adev)
2921 {
2922         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2923         struct amdgpu_bo *bo, *tmp;
2924         struct dma_fence *fence = NULL, *next = NULL;
2925         long r = 1;
2926         int i = 0;
2927         long tmo;
2928
2929         if (amdgpu_sriov_runtime(adev))
2930                 tmo = msecs_to_jiffies(amdgpu_lockup_timeout);
2931         else
2932                 tmo = msecs_to_jiffies(100);
2933
2934         DRM_INFO("recover vram bo from shadow start\n");
2935         mutex_lock(&adev->shadow_list_lock);
2936         list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2937                 next = NULL;
2938                 amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next);
2939                 if (fence) {
2940                         r = dma_fence_wait_timeout(fence, false, tmo);
2941                         if (r == 0)
2942                                 pr_err("wait fence %p[%d] timeout\n", fence, i);
2943                         else if (r < 0)
2944                                 pr_err("wait fence %p[%d] interrupted\n", fence, i);
2945                         if (r < 1) {
2946                                 dma_fence_put(fence);
2947                                 fence = next;
2948                                 break;
2949                         }
2950                         i++;
2951                 }
2952
2953                 dma_fence_put(fence);
2954                 fence = next;
2955         }
2956         mutex_unlock(&adev->shadow_list_lock);
2957
2958         if (fence) {
2959                 r = dma_fence_wait_timeout(fence, false, tmo);
2960                 if (r == 0)
2961                         pr_err("wait fence %p[%d] timeout\n", fence, i);
2962                 else if (r < 0)
2963                         pr_err("wait fence %p[%d] interrupted\n", fence, i);
2964
2965         }
2966         dma_fence_put(fence);
2967
2968         if (r > 0)
2969                 DRM_INFO("recover vram bo from shadow done\n");
2970         else
2971                 DRM_ERROR("recover vram bo from shadow failed\n");
2972
2973         return (r > 0) ? 0 : 1;
2974 }
2975
2976 /**
2977  * amdgpu_device_reset - reset ASIC/GPU for bare-metal or passthrough
2978  *
2979  * @adev: amdgpu device pointer
2980  *
2981  * attempt to do soft-reset or full-reset and reinitialize Asic
2982  * return 0 means successed otherwise failed
2983  */
2984 static int amdgpu_device_reset(struct amdgpu_device *adev)
2985 {
2986         bool need_full_reset, vram_lost = 0;
2987         int r;
2988
2989         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
2990
2991         if (!need_full_reset) {
2992                 amdgpu_device_ip_pre_soft_reset(adev);
2993                 r = amdgpu_device_ip_soft_reset(adev);
2994                 amdgpu_device_ip_post_soft_reset(adev);
2995                 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
2996                         DRM_INFO("soft reset failed, will fallback to full reset!\n");
2997                         need_full_reset = true;
2998                 }
2999         }
3000
3001         if (need_full_reset) {
3002                 r = amdgpu_device_ip_suspend(adev);
3003
3004 retry:
3005                 r = amdgpu_asic_reset(adev);
3006                 /* post card */
3007                 amdgpu_atom_asic_init(adev->mode_info.atom_context);
3008
3009                 if (!r) {
3010                         dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
3011                         r = amdgpu_device_ip_resume_phase1(adev);
3012                         if (r)
3013                                 goto out;
3014
3015                         vram_lost = amdgpu_device_check_vram_lost(adev);
3016                         if (vram_lost) {
3017                                 DRM_ERROR("VRAM is lost!\n");
3018                                 atomic_inc(&adev->vram_lost_counter);
3019                         }
3020
3021                         r = amdgpu_gtt_mgr_recover(
3022                                 &adev->mman.bdev.man[TTM_PL_TT]);
3023                         if (r)
3024                                 goto out;
3025
3026                         r = amdgpu_device_ip_resume_phase2(adev);
3027                         if (r)
3028                                 goto out;
3029
3030                         if (vram_lost)
3031                                 amdgpu_device_fill_reset_magic(adev);
3032                 }
3033         }
3034
3035 out:
3036         if (!r) {
3037                 amdgpu_irq_gpu_reset_resume_helper(adev);
3038                 r = amdgpu_ib_ring_tests(adev);
3039                 if (r) {
3040                         dev_err(adev->dev, "ib ring test failed (%d).\n", r);
3041                         r = amdgpu_device_ip_suspend(adev);
3042                         need_full_reset = true;
3043                         goto retry;
3044                 }
3045         }
3046
3047         if (!r && ((need_full_reset && !(adev->flags & AMD_IS_APU)) || vram_lost))
3048                 r = amdgpu_device_handle_vram_lost(adev);
3049
3050         return r;
3051 }
3052
3053 /**
3054  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
3055  *
3056  * @adev: amdgpu device pointer
3057  *
3058  * do VF FLR and reinitialize Asic
3059  * return 0 means successed otherwise failed
3060  */
3061 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
3062                                      bool from_hypervisor)
3063 {
3064         int r;
3065
3066         if (from_hypervisor)
3067                 r = amdgpu_virt_request_full_gpu(adev, true);
3068         else
3069                 r = amdgpu_virt_reset_gpu(adev);
3070         if (r)
3071                 return r;
3072
3073         /* Resume IP prior to SMC */
3074         r = amdgpu_device_ip_reinit_early_sriov(adev);
3075         if (r)
3076                 goto error;
3077
3078         /* we need recover gart prior to run SMC/CP/SDMA resume */
3079         amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
3080
3081         /* now we are okay to resume SMC/CP/SDMA */
3082         r = amdgpu_device_ip_reinit_late_sriov(adev);
3083         amdgpu_virt_release_full_gpu(adev, true);
3084         if (r)
3085                 goto error;
3086
3087         amdgpu_irq_gpu_reset_resume_helper(adev);
3088         r = amdgpu_ib_ring_tests(adev);
3089
3090         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
3091                 atomic_inc(&adev->vram_lost_counter);
3092                 r = amdgpu_device_handle_vram_lost(adev);
3093         }
3094
3095 error:
3096
3097         return r;
3098 }
3099
3100 /**
3101  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
3102  *
3103  * @adev: amdgpu device pointer
3104  * @job: which job trigger hang
3105  * @force forces reset regardless of amdgpu_gpu_recovery
3106  *
3107  * Attempt to reset the GPU if it has hung (all asics).
3108  * Returns 0 for success or an error on failure.
3109  */
3110 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
3111                               struct amdgpu_job *job, bool force)
3112 {
3113         struct drm_atomic_state *state = NULL;
3114         int i, r, resched;
3115
3116         if (!force && !amdgpu_device_ip_check_soft_reset(adev)) {
3117                 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
3118                 return 0;
3119         }
3120
3121         if (!force && (amdgpu_gpu_recovery == 0 ||
3122                         (amdgpu_gpu_recovery == -1  && !amdgpu_sriov_vf(adev)))) {
3123                 DRM_INFO("GPU recovery disabled.\n");
3124                 return 0;
3125         }
3126
3127         dev_info(adev->dev, "GPU reset begin!\n");
3128
3129         mutex_lock(&adev->lock_reset);
3130         atomic_inc(&adev->gpu_reset_counter);
3131         adev->in_gpu_reset = 1;
3132
3133         /* block TTM */
3134         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3135
3136         /* store modesetting */
3137         if (amdgpu_device_has_dc_support(adev))
3138                 state = drm_atomic_helper_suspend(adev->ddev);
3139
3140         /* block all schedulers and reset given job's ring */
3141         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3142                 struct amdgpu_ring *ring = adev->rings[i];
3143
3144                 if (!ring || !ring->sched.thread)
3145                         continue;
3146
3147                 kthread_park(ring->sched.thread);
3148
3149                 if (job && job->ring->idx != i)
3150                         continue;
3151
3152                 drm_sched_hw_job_reset(&ring->sched, &job->base);
3153
3154                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
3155                 amdgpu_fence_driver_force_completion(ring);
3156         }
3157
3158         if (amdgpu_sriov_vf(adev))
3159                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
3160         else
3161                 r = amdgpu_device_reset(adev);
3162
3163         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3164                 struct amdgpu_ring *ring = adev->rings[i];
3165
3166                 if (!ring || !ring->sched.thread)
3167                         continue;
3168
3169                 /* only need recovery sched of the given job's ring
3170                  * or all rings (in the case @job is NULL)
3171                  * after above amdgpu_reset accomplished
3172                  */
3173                 if ((!job || job->ring->idx == i) && !r)
3174                         drm_sched_job_recovery(&ring->sched);
3175
3176                 kthread_unpark(ring->sched.thread);
3177         }
3178
3179         if (amdgpu_device_has_dc_support(adev)) {
3180                 if (drm_atomic_helper_resume(adev->ddev, state))
3181                         dev_info(adev->dev, "drm resume failed:%d\n", r);
3182         } else {
3183                 drm_helper_resume_force_mode(adev->ddev);
3184         }
3185
3186         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
3187
3188         if (r) {
3189                 /* bad news, how to tell it to userspace ? */
3190                 dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
3191                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
3192         } else {
3193                 dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
3194         }
3195
3196         amdgpu_vf_error_trans_all(adev);
3197         adev->in_gpu_reset = 0;
3198         mutex_unlock(&adev->lock_reset);
3199         return r;
3200 }
3201
3202 /**
3203  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
3204  *
3205  * @adev: amdgpu_device pointer
3206  *
3207  * Fetchs and stores in the driver the PCIE capabilities (gen speed
3208  * and lanes) of the slot the device is in. Handles APUs and
3209  * virtualized environments where PCIE config space may not be available.
3210  */
3211 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
3212 {
3213         u32 mask;
3214         int ret;
3215
3216         if (amdgpu_pcie_gen_cap)
3217                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
3218
3219         if (amdgpu_pcie_lane_cap)
3220                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
3221
3222         /* covers APUs as well */
3223         if (pci_is_root_bus(adev->pdev->bus)) {
3224                 if (adev->pm.pcie_gen_mask == 0)
3225                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3226                 if (adev->pm.pcie_mlw_mask == 0)
3227                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3228                 return;
3229         }
3230
3231         if (adev->pm.pcie_gen_mask == 0) {
3232                 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
3233                 if (!ret) {
3234                         adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3235                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3236                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
3237
3238                         if (mask & DRM_PCIE_SPEED_25)
3239                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
3240                         if (mask & DRM_PCIE_SPEED_50)
3241                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
3242                         if (mask & DRM_PCIE_SPEED_80)
3243                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
3244                 } else {
3245                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3246                 }
3247         }
3248         if (adev->pm.pcie_mlw_mask == 0) {
3249                 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
3250                 if (!ret) {
3251                         switch (mask) {
3252                         case 32:
3253                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
3254                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3255                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3256                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3257                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3258                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3259                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3260                                 break;
3261                         case 16:
3262                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3263                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3264                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3265                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3266                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3267                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3268                                 break;
3269                         case 12:
3270                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3271                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3272                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3273                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3274                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3275                                 break;
3276                         case 8:
3277                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3278                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3279                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3280                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3281                                 break;
3282                         case 4:
3283                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3284                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3285                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3286                                 break;
3287                         case 2:
3288                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3289                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3290                                 break;
3291                         case 1:
3292                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
3293                                 break;
3294                         default:
3295                                 break;
3296                         }
3297                 } else {
3298                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3299                 }
3300         }
3301 }
3302
This page took 0.236341 seconds and 4 git commands to generate.