2 * Copyright 2018 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/debugfs.h>
25 #include <linux/list.h>
26 #include <linux/module.h>
27 #include <linux/uaccess.h>
28 #include <linux/reboot.h>
29 #include <linux/syscalls.h>
30 #include <linux/pm_runtime.h>
31 #include <linux/list_sort.h>
34 #include "amdgpu_ras.h"
35 #include "amdgpu_atomfirmware.h"
36 #include "amdgpu_xgmi.h"
37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
38 #include "nbio_v4_3.h"
39 #include "nbif_v6_3_1.h"
40 #include "nbio_v7_9.h"
42 #include "amdgpu_reset.h"
43 #include "amdgpu_psp.h"
45 #ifdef CONFIG_X86_MCE_AMD
48 static bool notifier_registered;
50 static const char *RAS_FS_NAME = "ras";
52 const char *ras_error_string[] = {
56 "multi_uncorrectable",
60 const char *ras_block_string[] = {
82 const char *ras_mca_block_string[] = {
89 struct amdgpu_ras_block_list {
91 struct list_head node;
93 struct amdgpu_ras_block_object *ras_obj;
96 const char *get_ras_block_str(struct ras_common_if *ras_block)
101 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT ||
102 ras_block->block >= ARRAY_SIZE(ras_block_string))
103 return "OUT OF RANGE";
105 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
106 return ras_mca_block_string[ras_block->sub_block_index];
108 return ras_block_string[ras_block->block];
111 #define ras_block_str(_BLOCK_) \
112 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
114 #define ras_err_str(i) (ras_error_string[ffs(i)])
116 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
118 /* inject address is 52 bits */
119 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52)
121 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */
122 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL)
124 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 300 //ms
126 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms
128 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100
130 enum amdgpu_ras_retire_page_reservation {
131 AMDGPU_RAS_RETIRE_PAGE_RESERVED,
132 AMDGPU_RAS_RETIRE_PAGE_PENDING,
133 AMDGPU_RAS_RETIRE_PAGE_FAULT,
136 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
138 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
140 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
142 #ifdef CONFIG_X86_MCE_AMD
143 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
144 struct mce_notifier_adev_list {
145 struct amdgpu_device *devs[MAX_GPU_INSTANCE];
148 static struct mce_notifier_adev_list mce_adev_list;
151 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
153 if (adev && amdgpu_ras_get_context(adev))
154 amdgpu_ras_get_context(adev)->error_query_ready = ready;
157 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
159 if (adev && amdgpu_ras_get_context(adev))
160 return amdgpu_ras_get_context(adev)->error_query_ready;
165 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
167 struct ras_err_data err_data;
168 struct eeprom_table_record err_rec;
171 if ((address >= adev->gmc.mc_vram_size) ||
172 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
174 "RAS WARN: input address 0x%llx is invalid.\n",
179 if (amdgpu_ras_check_bad_page(adev, address)) {
181 "RAS WARN: 0x%llx has already been marked as bad page!\n",
186 ret = amdgpu_ras_error_data_init(&err_data);
190 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
191 err_data.err_addr = &err_rec;
192 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0);
194 if (amdgpu_bad_page_threshold != 0) {
195 amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
196 err_data.err_addr_cnt, false);
197 amdgpu_ras_save_bad_pages(adev, NULL);
200 amdgpu_ras_error_data_fini(&err_data);
202 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
203 dev_warn(adev->dev, "Clear EEPROM:\n");
204 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
209 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
210 size_t size, loff_t *pos)
212 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
213 struct ras_query_if info = {
219 if (amdgpu_ras_query_error_status(obj->adev, &info))
222 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
223 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
224 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
225 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
226 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
229 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
231 "ce", info.ce_count);
236 s = min_t(u64, s, size);
239 if (copy_to_user(buf, &val[*pos], s))
247 static const struct file_operations amdgpu_ras_debugfs_ops = {
248 .owner = THIS_MODULE,
249 .read = amdgpu_ras_debugfs_read,
251 .llseek = default_llseek
254 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
258 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
260 if (strcmp(name, ras_block_string[i]) == 0)
266 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
267 const char __user *buf, size_t size,
268 loff_t *pos, struct ras_debug_if *data)
270 ssize_t s = min_t(u64, 64, size);
278 /* default value is 0 if the mask is not set by user */
279 u32 instance_mask = 0;
285 memset(str, 0, sizeof(str));
286 memset(data, 0, sizeof(*data));
288 if (copy_from_user(str, buf, s))
291 if (sscanf(str, "disable %32s", block_name) == 1)
293 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
295 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
297 else if (strstr(str, "retire_page") != NULL)
299 else if (str[0] && str[1] && str[2] && str[3])
300 /* ascii string, but commands are not matched. */
305 if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
306 sscanf(str, "%*s %llu", &address) != 1)
310 data->inject.address = address;
315 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
318 data->head.block = block_id;
319 /* only ue, ce and poison errors are supported */
320 if (!memcmp("ue", err, 2))
321 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
322 else if (!memcmp("ce", err, 2))
323 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
324 else if (!memcmp("poison", err, 6))
325 data->head.type = AMDGPU_RAS_ERROR__POISON;
332 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x",
333 &sub_block, &address, &value, &instance_mask) != 4 &&
334 sscanf(str, "%*s %*s %*s %u %llu %llu %u",
335 &sub_block, &address, &value, &instance_mask) != 4 &&
336 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
337 &sub_block, &address, &value) != 3 &&
338 sscanf(str, "%*s %*s %*s %u %llu %llu",
339 &sub_block, &address, &value) != 3)
341 data->head.sub_block_index = sub_block;
342 data->inject.address = address;
343 data->inject.value = value;
344 data->inject.instance_mask = instance_mask;
347 if (size < sizeof(*data))
350 if (copy_from_user(data, buf, sizeof(*data)))
357 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev,
358 struct ras_debug_if *data)
360 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
361 uint32_t mask, inst_mask = data->inject.instance_mask;
363 /* no need to set instance mask if there is only one instance */
364 if (num_xcc <= 1 && inst_mask) {
365 data->inject.instance_mask = 0;
367 "RAS inject mask(0x%x) isn't supported and force it to 0.\n",
373 switch (data->head.block) {
374 case AMDGPU_RAS_BLOCK__GFX:
375 mask = GENMASK(num_xcc - 1, 0);
377 case AMDGPU_RAS_BLOCK__SDMA:
378 mask = GENMASK(adev->sdma.num_instances - 1, 0);
380 case AMDGPU_RAS_BLOCK__VCN:
381 case AMDGPU_RAS_BLOCK__JPEG:
382 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0);
389 /* remove invalid bits in instance mask */
390 data->inject.instance_mask &= mask;
391 if (inst_mask != data->inject.instance_mask)
393 "Adjust RAS inject mask 0x%x to 0x%x\n",
394 inst_mask, data->inject.instance_mask);
398 * DOC: AMDGPU RAS debugfs control interface
400 * The control interface accepts struct ras_debug_if which has two members.
402 * First member: ras_debug_if::head or ras_debug_if::inject.
404 * head is used to indicate which IP block will be under control.
406 * head has four members, they are block, type, sub_block_index, name.
407 * block: which IP will be under control.
408 * type: what kind of error will be enabled/disabled/injected.
409 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
410 * name: the name of IP.
412 * inject has three more members than head, they are address, value and mask.
413 * As their names indicate, inject operation will write the
414 * value to the address.
416 * The second member: struct ras_debug_if::op.
417 * It has three kinds of operations.
419 * - 0: disable RAS on the block. Take ::head as its data.
420 * - 1: enable RAS on the block. Take ::head as its data.
421 * - 2: inject errors on the block. Take ::inject as its data.
423 * How to use the interface?
427 * Copy the struct ras_debug_if in your code and initialize it.
428 * Write the struct to the control interface.
432 * .. code-block:: bash
434 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
435 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
436 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
438 * Where N, is the card which you want to affect.
440 * "disable" requires only the block.
441 * "enable" requires the block and error type.
442 * "inject" requires the block, error type, address, and value.
444 * The block is one of: umc, sdma, gfx, etc.
445 * see ras_block_string[] for details
447 * The error type is one of: ue, ce and poison where,
448 * ue is multi-uncorrectable
449 * ce is single-correctable
452 * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
453 * The address and value are hexadecimal numbers, leading 0x is optional.
454 * The mask means instance mask, is optional, default value is 0x1.
458 * .. code-block:: bash
460 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
461 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl
462 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
464 * How to check the result of the operation?
466 * To check disable/enable, see "ras" features at,
467 * /sys/class/drm/card[0/1/2...]/device/ras/features
469 * To check inject, see the corresponding error count at,
470 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
473 * Operations are only allowed on blocks which are supported.
474 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
475 * to see which blocks support RAS on a particular asic.
478 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
479 const char __user *buf,
480 size_t size, loff_t *pos)
482 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
483 struct ras_debug_if data;
486 if (!amdgpu_ras_get_error_query_ready(adev)) {
487 dev_warn(adev->dev, "RAS WARN: error injection "
488 "currently inaccessible\n");
492 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
497 ret = amdgpu_reserve_page_direct(adev, data.inject.address);
504 if (!amdgpu_ras_is_supported(adev, data.head.block))
509 ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
512 ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
515 if ((data.inject.address >= adev->gmc.mc_vram_size &&
516 adev->gmc.mc_vram_size) ||
517 (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
518 dev_warn(adev->dev, "RAS WARN: input address "
519 "0x%llx is invalid.",
520 data.inject.address);
525 /* umc ce/ue error injection for a bad page is not allowed */
526 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
527 amdgpu_ras_check_bad_page(adev, data.inject.address)) {
528 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
529 "already been marked as bad!\n",
530 data.inject.address);
534 amdgpu_ras_instance_mask_check(adev, &data);
536 /* data.inject.address is offset instead of absolute gpu address */
537 ret = amdgpu_ras_error_inject(adev, &data.inject);
551 * DOC: AMDGPU RAS debugfs EEPROM table reset interface
553 * Some boards contain an EEPROM which is used to persistently store a list of
554 * bad pages which experiences ECC errors in vram. This interface provides
555 * a way to reset the EEPROM, e.g., after testing error injection.
559 * .. code-block:: bash
561 * echo 1 > ../ras/ras_eeprom_reset
563 * will reset EEPROM table to 0 entries.
566 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
567 const char __user *buf,
568 size_t size, loff_t *pos)
570 struct amdgpu_device *adev =
571 (struct amdgpu_device *)file_inode(f)->i_private;
574 ret = amdgpu_ras_eeprom_reset_table(
575 &(amdgpu_ras_get_context(adev)->eeprom_control));
578 /* Something was written to EEPROM.
580 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
587 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
588 .owner = THIS_MODULE,
590 .write = amdgpu_ras_debugfs_ctrl_write,
591 .llseek = default_llseek
594 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
595 .owner = THIS_MODULE,
597 .write = amdgpu_ras_debugfs_eeprom_write,
598 .llseek = default_llseek
602 * DOC: AMDGPU RAS sysfs Error Count Interface
604 * It allows the user to read the error count for each IP block on the gpu through
605 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
607 * It outputs the multiple lines which report the uncorrected (ue) and corrected
610 * The format of one line is below,
616 * .. code-block:: bash
622 static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
623 struct device_attribute *attr, char *buf)
625 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
626 struct ras_query_if info = {
630 if (!amdgpu_ras_get_error_query_ready(obj->adev))
631 return sysfs_emit(buf, "Query currently inaccessible\n");
633 if (amdgpu_ras_query_error_status(obj->adev, &info))
636 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
637 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
638 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
639 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
642 if (info.head.block == AMDGPU_RAS_BLOCK__UMC)
643 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
644 "ce", info.ce_count, "de", info.de_count);
646 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
647 "ce", info.ce_count);
652 #define get_obj(obj) do { (obj)->use++; } while (0)
653 #define alive_obj(obj) ((obj)->use)
655 static inline void put_obj(struct ras_manager *obj)
657 if (obj && (--obj->use == 0)) {
658 list_del(&obj->node);
659 amdgpu_ras_error_data_fini(&obj->err_data);
662 if (obj && (obj->use < 0))
663 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
666 /* make one obj and return it. */
667 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
668 struct ras_common_if *head)
670 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
671 struct ras_manager *obj;
673 if (!adev->ras_enabled || !con)
676 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
679 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
680 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
683 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
685 obj = &con->objs[head->block];
687 /* already exist. return obj? */
691 if (amdgpu_ras_error_data_init(&obj->err_data))
696 list_add(&obj->node, &con->head);
702 /* return an obj equal to head, or the first when head is NULL */
703 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
704 struct ras_common_if *head)
706 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
707 struct ras_manager *obj;
710 if (!adev->ras_enabled || !con)
714 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
717 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
718 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
721 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
723 obj = &con->objs[head->block];
728 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
739 /* feature ctl begin */
740 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
741 struct ras_common_if *head)
743 return adev->ras_hw_enabled & BIT(head->block);
746 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
747 struct ras_common_if *head)
749 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
751 return con->features & BIT(head->block);
755 * if obj is not created, then create one.
756 * set feature enable flag.
758 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
759 struct ras_common_if *head, int enable)
761 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
762 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
764 /* If hardware does not support ras, then do not create obj.
765 * But if hardware support ras, we can create the obj.
766 * Ras framework checks con->hw_supported to see if it need do
767 * corresponding initialization.
768 * IP checks con->support to see if it need disable ras.
770 if (!amdgpu_ras_is_feature_allowed(adev, head))
775 obj = amdgpu_ras_create_obj(adev, head);
779 /* In case we create obj somewhere else */
782 con->features |= BIT(head->block);
784 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
785 con->features &= ~BIT(head->block);
793 /* wrapper of psp_ras_enable_features */
794 int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
795 struct ras_common_if *head, bool enable)
797 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
798 union ta_ras_cmd_input *info;
804 /* For non-gfx ip, do not enable ras feature if it is not allowed */
805 /* For gfx ip, regardless of feature support status, */
806 /* Force issue enable or disable ras feature commands */
807 if (head->block != AMDGPU_RAS_BLOCK__GFX &&
808 !amdgpu_ras_is_feature_allowed(adev, head))
811 /* Only enable gfx ras feature from host side */
812 if (head->block == AMDGPU_RAS_BLOCK__GFX &&
813 !amdgpu_sriov_vf(adev) &&
814 !amdgpu_ras_intr_triggered()) {
815 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
820 info->disable_features = (struct ta_ras_disable_features_input) {
821 .block_id = amdgpu_ras_block_to_ta(head->block),
822 .error_type = amdgpu_ras_error_to_ta(head->type),
825 info->enable_features = (struct ta_ras_enable_features_input) {
826 .block_id = amdgpu_ras_block_to_ta(head->block),
827 .error_type = amdgpu_ras_error_to_ta(head->type),
831 ret = psp_ras_enable_features(&adev->psp, info, enable);
833 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
834 enable ? "enable":"disable",
835 get_ras_block_str(head),
836 amdgpu_ras_is_poison_mode_supported(adev), ret);
845 __amdgpu_ras_feature_enable(adev, head, enable);
850 /* Only used in device probe stage and called only once. */
851 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
852 struct ras_common_if *head, bool enable)
854 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
860 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
862 /* There is no harm to issue a ras TA cmd regardless of
863 * the currecnt ras state.
864 * If current state == target state, it will do nothing
865 * But sometimes it requests driver to reset and repost
866 * with error code -EAGAIN.
868 ret = amdgpu_ras_feature_enable(adev, head, 1);
869 /* With old ras TA, we might fail to enable ras.
870 * Log it and just setup the object.
871 * TODO need remove this WA in the future.
873 if (ret == -EINVAL) {
874 ret = __amdgpu_ras_feature_enable(adev, head, 1);
877 "RAS INFO: %s setup object\n",
878 get_ras_block_str(head));
881 /* setup the object then issue a ras TA disable cmd.*/
882 ret = __amdgpu_ras_feature_enable(adev, head, 1);
886 /* gfx block ras disable cmd must send to ras-ta */
887 if (head->block == AMDGPU_RAS_BLOCK__GFX)
888 con->features |= BIT(head->block);
890 ret = amdgpu_ras_feature_enable(adev, head, 0);
892 /* clean gfx block ras features flag */
893 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
894 con->features &= ~BIT(head->block);
897 ret = amdgpu_ras_feature_enable(adev, head, enable);
902 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
905 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
906 struct ras_manager *obj, *tmp;
908 list_for_each_entry_safe(obj, tmp, &con->head, node) {
910 * aka just release the obj and corresponding flags
913 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
916 if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
921 return con->features;
924 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
927 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
929 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
931 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
932 struct ras_common_if head = {
934 .type = default_ras_type,
935 .sub_block_index = 0,
938 if (i == AMDGPU_RAS_BLOCK__MCA)
943 * bypass psp. vbios enable ras for us.
944 * so just create the obj
946 if (__amdgpu_ras_feature_enable(adev, &head, 1))
949 if (amdgpu_ras_feature_enable(adev, &head, 1))
954 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
955 struct ras_common_if head = {
956 .block = AMDGPU_RAS_BLOCK__MCA,
957 .type = default_ras_type,
958 .sub_block_index = i,
963 * bypass psp. vbios enable ras for us.
964 * so just create the obj
966 if (__amdgpu_ras_feature_enable(adev, &head, 1))
969 if (amdgpu_ras_feature_enable(adev, &head, 1))
974 return con->features;
976 /* feature ctl end */
978 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
979 enum amdgpu_ras_block block)
984 if (block_obj->ras_comm.block == block)
990 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
991 enum amdgpu_ras_block block, uint32_t sub_block_index)
993 struct amdgpu_ras_block_list *node, *tmp;
994 struct amdgpu_ras_block_object *obj;
996 if (block >= AMDGPU_RAS_BLOCK__LAST)
999 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
1000 if (!node->ras_obj) {
1001 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
1005 obj = node->ras_obj;
1006 if (obj->ras_block_match) {
1007 if (obj->ras_block_match(obj, block, sub_block_index) == 0)
1010 if (amdgpu_ras_block_match_default(obj, block) == 0)
1018 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
1020 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
1024 * choosing right query method according to
1025 * whether smu support query error information
1027 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
1028 if (ret == -EOPNOTSUPP) {
1029 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1030 adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
1031 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
1033 /* umc query_ras_error_address is also responsible for clearing
1036 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1037 adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
1038 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
1040 if (adev->umc.ras &&
1041 adev->umc.ras->ecc_info_query_ras_error_count)
1042 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
1044 if (adev->umc.ras &&
1045 adev->umc.ras->ecc_info_query_ras_error_address)
1046 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
1050 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
1051 struct ras_manager *ras_mgr,
1052 struct ras_err_data *err_data,
1053 struct ras_query_context *qctx,
1054 const char *blk_name,
1058 struct amdgpu_smuio_mcm_config_info *mcm_info;
1059 struct ras_err_node *err_node;
1060 struct ras_err_info *err_info;
1061 u64 event_id = qctx->evid.event_id;
1064 for_each_ras_error(err_node, err_data) {
1065 err_info = &err_node->err_info;
1066 mcm_info = &err_info->mcm_info;
1067 if (err_info->ue_count) {
1068 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1069 "%lld new uncorrectable hardware errors detected in %s block\n",
1070 mcm_info->socket_id,
1077 for_each_ras_error(err_node, &ras_mgr->err_data) {
1078 err_info = &err_node->err_info;
1079 mcm_info = &err_info->mcm_info;
1080 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1081 "%lld uncorrectable hardware errors detected in total in %s block\n",
1082 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name);
1087 for_each_ras_error(err_node, err_data) {
1088 err_info = &err_node->err_info;
1089 mcm_info = &err_info->mcm_info;
1090 if (err_info->de_count) {
1091 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1092 "%lld new deferred hardware errors detected in %s block\n",
1093 mcm_info->socket_id,
1100 for_each_ras_error(err_node, &ras_mgr->err_data) {
1101 err_info = &err_node->err_info;
1102 mcm_info = &err_info->mcm_info;
1103 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1104 "%lld deferred hardware errors detected in total in %s block\n",
1105 mcm_info->socket_id, mcm_info->die_id,
1106 err_info->de_count, blk_name);
1109 for_each_ras_error(err_node, err_data) {
1110 err_info = &err_node->err_info;
1111 mcm_info = &err_info->mcm_info;
1112 if (err_info->ce_count) {
1113 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1114 "%lld new correctable hardware errors detected in %s block\n",
1115 mcm_info->socket_id,
1122 for_each_ras_error(err_node, &ras_mgr->err_data) {
1123 err_info = &err_node->err_info;
1124 mcm_info = &err_info->mcm_info;
1125 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1126 "%lld correctable hardware errors detected in total in %s block\n",
1127 mcm_info->socket_id, mcm_info->die_id,
1128 err_info->ce_count, blk_name);
1134 static inline bool err_data_has_source_info(struct ras_err_data *data)
1136 return !list_empty(&data->err_node_list);
1139 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
1140 struct ras_query_if *query_if,
1141 struct ras_err_data *err_data,
1142 struct ras_query_context *qctx)
1144 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head);
1145 const char *blk_name = get_ras_block_str(&query_if->head);
1146 u64 event_id = qctx->evid.event_id;
1148 if (err_data->ce_count) {
1149 if (err_data_has_source_info(err_data)) {
1150 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1151 blk_name, false, false);
1152 } else if (!adev->aid_mask &&
1153 adev->smuio.funcs &&
1154 adev->smuio.funcs->get_socket_id &&
1155 adev->smuio.funcs->get_die_id) {
1156 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1157 "%ld correctable hardware errors "
1158 "detected in %s block\n",
1159 adev->smuio.funcs->get_socket_id(adev),
1160 adev->smuio.funcs->get_die_id(adev),
1161 ras_mgr->err_data.ce_count,
1164 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors "
1165 "detected in %s block\n",
1166 ras_mgr->err_data.ce_count,
1171 if (err_data->ue_count) {
1172 if (err_data_has_source_info(err_data)) {
1173 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1174 blk_name, true, false);
1175 } else if (!adev->aid_mask &&
1176 adev->smuio.funcs &&
1177 adev->smuio.funcs->get_socket_id &&
1178 adev->smuio.funcs->get_die_id) {
1179 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1180 "%ld uncorrectable hardware errors "
1181 "detected in %s block\n",
1182 adev->smuio.funcs->get_socket_id(adev),
1183 adev->smuio.funcs->get_die_id(adev),
1184 ras_mgr->err_data.ue_count,
1187 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors "
1188 "detected in %s block\n",
1189 ras_mgr->err_data.ue_count,
1194 if (err_data->de_count) {
1195 if (err_data_has_source_info(err_data)) {
1196 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1197 blk_name, false, true);
1198 } else if (!adev->aid_mask &&
1199 adev->smuio.funcs &&
1200 adev->smuio.funcs->get_socket_id &&
1201 adev->smuio.funcs->get_die_id) {
1202 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1203 "%ld deferred hardware errors "
1204 "detected in %s block\n",
1205 adev->smuio.funcs->get_socket_id(adev),
1206 adev->smuio.funcs->get_die_id(adev),
1207 ras_mgr->err_data.de_count,
1210 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors "
1211 "detected in %s block\n",
1212 ras_mgr->err_data.de_count,
1218 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev,
1219 struct ras_query_if *query_if,
1220 struct ras_err_data *err_data,
1221 struct ras_query_context *qctx)
1223 unsigned long new_ue, new_ce, new_de;
1224 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head);
1225 const char *blk_name = get_ras_block_str(&query_if->head);
1226 u64 event_id = qctx->evid.event_id;
1228 new_ce = err_data->ce_count - obj->err_data.ce_count;
1229 new_ue = err_data->ue_count - obj->err_data.ue_count;
1230 new_de = err_data->de_count - obj->err_data.de_count;
1233 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors "
1234 "detected in %s block\n",
1240 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors "
1241 "detected in %s block\n",
1247 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors "
1248 "detected in %s block\n",
1254 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data)
1256 struct ras_err_node *err_node;
1257 struct ras_err_info *err_info;
1259 if (err_data_has_source_info(err_data)) {
1260 for_each_ras_error(err_node, err_data) {
1261 err_info = &err_node->err_info;
1262 amdgpu_ras_error_statistic_de_count(&obj->err_data,
1263 &err_info->mcm_info, err_info->de_count);
1264 amdgpu_ras_error_statistic_ce_count(&obj->err_data,
1265 &err_info->mcm_info, err_info->ce_count);
1266 amdgpu_ras_error_statistic_ue_count(&obj->err_data,
1267 &err_info->mcm_info, err_info->ue_count);
1270 /* for legacy asic path which doesn't has error source info */
1271 obj->err_data.ue_count += err_data->ue_count;
1272 obj->err_data.ce_count += err_data->ce_count;
1273 obj->err_data.de_count += err_data->de_count;
1277 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj,
1278 struct ras_err_data *err_data)
1280 /* Host reports absolute counts */
1281 obj->err_data.ue_count = err_data->ue_count;
1282 obj->err_data.ce_count = err_data->ce_count;
1283 obj->err_data.de_count = err_data->de_count;
1286 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1288 struct ras_common_if head;
1290 memset(&head, 0, sizeof(head));
1293 return amdgpu_ras_find_obj(adev, &head);
1296 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1297 const struct aca_info *aca_info, void *data)
1299 struct ras_manager *obj;
1301 /* in resume phase, no need to create aca fs node */
1302 if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
1305 obj = get_ras_manager(adev, blk);
1309 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data);
1312 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1314 struct ras_manager *obj;
1316 obj = get_ras_manager(adev, blk);
1320 amdgpu_aca_remove_handle(&obj->aca_handle);
1325 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1326 enum aca_error_type type, struct ras_err_data *err_data,
1327 struct ras_query_context *qctx)
1329 struct ras_manager *obj;
1331 obj = get_ras_manager(adev, blk);
1335 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx);
1338 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr,
1339 struct aca_handle *handle, char *buf, void *data)
1341 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle);
1342 struct ras_query_if info = {
1346 if (!amdgpu_ras_get_error_query_ready(obj->adev))
1347 return sysfs_emit(buf, "Query currently inaccessible\n");
1349 if (amdgpu_ras_query_error_status(obj->adev, &info))
1352 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
1353 "ce", info.ce_count, "de", info.de_count);
1356 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
1357 struct ras_query_if *info,
1358 struct ras_err_data *err_data,
1359 struct ras_query_context *qctx,
1360 unsigned int error_query_mode)
1362 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT;
1363 struct amdgpu_ras_block_object *block_obj = NULL;
1366 if (blk == AMDGPU_RAS_BLOCK_COUNT)
1369 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY)
1372 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1373 return amdgpu_virt_req_ras_err_count(adev, blk, err_data);
1374 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
1375 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
1376 amdgpu_ras_get_ecc_info(adev, err_data);
1378 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
1379 if (!block_obj || !block_obj->hw_ops) {
1380 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1381 get_ras_block_str(&info->head));
1385 if (block_obj->hw_ops->query_ras_error_count)
1386 block_obj->hw_ops->query_ras_error_count(adev, err_data);
1388 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
1389 (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
1390 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
1391 if (block_obj->hw_ops->query_ras_error_status)
1392 block_obj->hw_ops->query_ras_error_status(adev);
1396 if (amdgpu_aca_is_enabled(adev)) {
1397 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx);
1401 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx);
1405 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx);
1409 /* FIXME: add code to check return value later */
1410 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx);
1411 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx);
1418 /* query/inject/cure begin */
1419 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev,
1420 struct ras_query_if *info,
1421 enum ras_event_type type)
1423 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1424 struct ras_err_data err_data;
1425 struct ras_query_context qctx;
1426 unsigned int error_query_mode;
1432 ret = amdgpu_ras_error_data_init(&err_data);
1436 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode))
1439 memset(&qctx, 0, sizeof(qctx));
1440 qctx.evid.type = type;
1441 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type);
1443 if (!down_read_trylock(&adev->reset_domain->sem)) {
1445 goto out_fini_err_data;
1448 ret = amdgpu_ras_query_error_status_helper(adev, info,
1452 up_read(&adev->reset_domain->sem);
1454 goto out_fini_err_data;
1456 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1457 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data);
1458 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx);
1460 /* Host provides absolute error counts. First generate the report
1461 * using the previous VF internal count against new host count.
1462 * Then Update VF internal count.
1464 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx);
1465 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data);
1468 info->ue_count = obj->err_data.ue_count;
1469 info->ce_count = obj->err_data.ce_count;
1470 info->de_count = obj->err_data.de_count;
1473 amdgpu_ras_error_data_fini(&err_data);
1478 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info)
1480 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID);
1483 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev,
1484 enum amdgpu_ras_block block)
1486 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1487 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
1488 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
1490 if (!block_obj || !block_obj->hw_ops) {
1491 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1492 ras_block_str(block));
1496 if (!amdgpu_ras_is_supported(adev, block) ||
1497 !amdgpu_ras_get_aca_debug_mode(adev))
1500 /* skip ras error reset in gpu reset */
1501 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) &&
1502 ((smu_funcs && smu_funcs->set_debug_mode) ||
1503 (mca_funcs && mca_funcs->mca_set_debug_mode)))
1506 if (block_obj->hw_ops->reset_ras_error_count)
1507 block_obj->hw_ops->reset_ras_error_count(adev);
1512 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1513 enum amdgpu_ras_block block)
1515 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1517 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP)
1520 if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1521 (block == AMDGPU_RAS_BLOCK__MMHUB)) {
1522 if (block_obj->hw_ops->reset_ras_error_status)
1523 block_obj->hw_ops->reset_ras_error_status(adev);
1529 /* wrapper of psp_ras_trigger_error */
1530 int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1531 struct ras_inject_if *info)
1533 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1534 struct ta_ras_trigger_error_input block_info = {
1535 .block_id = amdgpu_ras_block_to_ta(info->head.block),
1536 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
1537 .sub_block_index = info->head.sub_block_index,
1538 .address = info->address,
1539 .value = info->value,
1542 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1544 info->head.sub_block_index);
1546 /* inject on guest isn't allowed, return success directly */
1547 if (amdgpu_sriov_vf(adev))
1553 if (!block_obj || !block_obj->hw_ops) {
1554 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1555 get_ras_block_str(&info->head));
1559 /* Calculate XGMI relative offset */
1560 if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1561 info->head.block != AMDGPU_RAS_BLOCK__GFX) {
1562 block_info.address =
1563 amdgpu_xgmi_get_relative_phy_addr(adev,
1564 block_info.address);
1567 if (block_obj->hw_ops->ras_error_inject) {
1568 if (info->head.block == AMDGPU_RAS_BLOCK__GFX)
1569 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask);
1570 else /* Special ras_error_inject is defined (e.g: xgmi) */
1571 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info,
1572 info->instance_mask);
1575 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask);
1579 dev_err(adev->dev, "ras inject %s failed %d\n",
1580 get_ras_block_str(&info->head), ret);
1586 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP
1587 * @adev: pointer to AMD GPU device
1588 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1589 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors.
1590 * @query_info: pointer to ras_query_if
1592 * Return 0 for query success or do nothing, otherwise return an error
1595 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev,
1596 unsigned long *ce_count,
1597 unsigned long *ue_count,
1598 struct ras_query_if *query_info)
1603 /* do nothing if query_info is not specified */
1606 ret = amdgpu_ras_query_error_status(adev, query_info);
1610 *ce_count += query_info->ce_count;
1611 *ue_count += query_info->ue_count;
1613 /* some hardware/IP supports read to clear
1614 * no need to explictly reset the err status after the query call */
1615 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
1616 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
1617 if (amdgpu_ras_reset_error_status(adev, query_info->head.block))
1619 "Failed to reset error counter and error status\n");
1626 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP
1627 * @adev: pointer to AMD GPU device
1628 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1629 * @ue_count: pointer to an integer to be set to the count of uncorrectible
1631 * @query_info: pointer to ras_query_if if the query request is only for
1632 * specific ip block; if info is NULL, then the qurey request is for
1633 * all the ip blocks that support query ras error counters/status
1635 * If set, @ce_count or @ue_count, count and return the corresponding
1636 * error counts in those integer pointers. Return 0 if the device
1637 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1639 int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1640 unsigned long *ce_count,
1641 unsigned long *ue_count,
1642 struct ras_query_if *query_info)
1644 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1645 struct ras_manager *obj;
1646 unsigned long ce, ue;
1649 if (!adev->ras_enabled || !con)
1652 /* Don't count since no reporting.
1654 if (!ce_count && !ue_count)
1660 /* query all the ip blocks that support ras query interface */
1661 list_for_each_entry(obj, &con->head, node) {
1662 struct ras_query_if info = {
1666 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info);
1669 /* query specific ip block */
1670 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info);
1684 /* query/inject/cure end */
1689 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1690 struct ras_badpage **bps, unsigned int *count);
1692 static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1695 case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
1697 case AMDGPU_RAS_RETIRE_PAGE_PENDING:
1699 case AMDGPU_RAS_RETIRE_PAGE_FAULT:
1706 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
1708 * It allows user to read the bad pages of vram on the gpu through
1709 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1711 * It outputs multiple lines, and each line stands for one gpu page.
1713 * The format of one line is below,
1714 * gpu pfn : gpu page size : flags
1716 * gpu pfn and gpu page size are printed in hex format.
1717 * flags can be one of below character,
1719 * R: reserved, this gpu page is reserved and not able to use.
1721 * P: pending for reserve, this gpu page is marked as bad, will be reserved
1722 * in next window of page_reserve.
1724 * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1728 * .. code-block:: bash
1730 * 0x00000001 : 0x00001000 : R
1731 * 0x00000002 : 0x00001000 : P
1735 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1736 struct kobject *kobj, struct bin_attribute *attr,
1737 char *buf, loff_t ppos, size_t count)
1739 struct amdgpu_ras *con =
1740 container_of(attr, struct amdgpu_ras, badpages_attr);
1741 struct amdgpu_device *adev = con->adev;
1742 const unsigned int element_size =
1743 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
1744 unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1745 unsigned int end = div64_ul(ppos + count - 1, element_size);
1747 struct ras_badpage *bps = NULL;
1748 unsigned int bps_count = 0;
1750 memset(buf, 0, count);
1752 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1755 for (; start < end && start < bps_count; start++)
1756 s += scnprintf(&buf[s], element_size + 1,
1757 "0x%08x : 0x%08x : %1s\n",
1760 amdgpu_ras_badpage_flags_str(bps[start].flags));
1767 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1768 struct device_attribute *attr, char *buf)
1770 struct amdgpu_ras *con =
1771 container_of(attr, struct amdgpu_ras, features_attr);
1773 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
1776 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev,
1777 struct device_attribute *attr, char *buf)
1779 struct amdgpu_ras *con =
1780 container_of(attr, struct amdgpu_ras, version_attr);
1781 return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version);
1784 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev,
1785 struct device_attribute *attr, char *buf)
1787 struct amdgpu_ras *con =
1788 container_of(attr, struct amdgpu_ras, schema_attr);
1789 return sysfs_emit(buf, "schema: 0x%x\n", con->schema);
1793 enum ras_event_type type;
1796 {RAS_EVENT_TYPE_FATAL, "Fatal Error"},
1797 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"},
1798 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"},
1801 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev,
1802 struct device_attribute *attr, char *buf)
1804 struct amdgpu_ras *con =
1805 container_of(attr, struct amdgpu_ras, event_state_attr);
1806 struct ras_event_manager *event_mgr = con->event_mgr;
1807 struct ras_event_state *event_state;
1813 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno));
1814 for (i = 0; i < ARRAY_SIZE(dump_event); i++) {
1815 event_state = &event_mgr->event_state[dump_event[i].type];
1816 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n",
1818 atomic64_read(&event_state->count),
1819 event_state->last_seqno);
1822 return (ssize_t)size;
1825 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1827 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1829 if (adev->dev->kobj.sd)
1830 sysfs_remove_file_from_group(&adev->dev->kobj,
1831 &con->badpages_attr.attr,
1835 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev)
1837 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1838 struct attribute *attrs[] = {
1839 &con->features_attr.attr,
1840 &con->version_attr.attr,
1841 &con->schema_attr.attr,
1842 &con->event_state_attr.attr,
1845 struct attribute_group group = {
1846 .name = RAS_FS_NAME,
1850 if (adev->dev->kobj.sd)
1851 sysfs_remove_group(&adev->dev->kobj, &group);
1856 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
1857 struct ras_common_if *head)
1859 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1861 if (amdgpu_aca_is_enabled(adev))
1864 if (!obj || obj->attr_inuse)
1869 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1870 "%s_err_count", head->name);
1872 obj->sysfs_attr = (struct device_attribute){
1874 .name = obj->fs_data.sysfs_name,
1877 .show = amdgpu_ras_sysfs_read,
1879 sysfs_attr_init(&obj->sysfs_attr.attr);
1881 if (sysfs_add_file_to_group(&adev->dev->kobj,
1882 &obj->sysfs_attr.attr,
1888 obj->attr_inuse = 1;
1893 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1894 struct ras_common_if *head)
1896 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1898 if (amdgpu_aca_is_enabled(adev))
1901 if (!obj || !obj->attr_inuse)
1904 if (adev->dev->kobj.sd)
1905 sysfs_remove_file_from_group(&adev->dev->kobj,
1906 &obj->sysfs_attr.attr,
1908 obj->attr_inuse = 0;
1914 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1916 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1917 struct ras_manager *obj, *tmp;
1919 list_for_each_entry_safe(obj, tmp, &con->head, node) {
1920 amdgpu_ras_sysfs_remove(adev, &obj->head);
1923 if (amdgpu_bad_page_threshold != 0)
1924 amdgpu_ras_sysfs_remove_bad_page_node(adev);
1926 amdgpu_ras_sysfs_remove_dev_attr_node(adev);
1933 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1935 * Normally when there is an uncorrectable error, the driver will reset
1936 * the GPU to recover. However, in the event of an unrecoverable error,
1937 * the driver provides an interface to reboot the system automatically
1940 * The following file in debugfs provides that interface:
1941 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1945 * .. code-block:: bash
1947 * echo true > .../ras/auto_reboot
1951 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
1953 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1954 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control;
1955 struct drm_minor *minor = adev_to_drm(adev)->primary;
1958 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1959 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1960 &amdgpu_ras_debugfs_ctrl_ops);
1961 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1962 &amdgpu_ras_debugfs_eeprom_ops);
1963 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1964 &con->bad_page_cnt_threshold);
1965 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs);
1966 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1967 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1968 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1969 &amdgpu_ras_debugfs_eeprom_size_ops);
1970 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1972 &amdgpu_ras_debugfs_eeprom_table_ops);
1973 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
1976 * After one uncorrectable error happens, usually GPU recovery will
1977 * be scheduled. But due to the known problem in GPU recovery failing
1978 * to bring GPU back, below interface provides one direct way to
1979 * user to reboot system automatically in such case within
1980 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1981 * will never be called.
1983 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
1986 * User could set this not to clean up hardware's error count register
1987 * of RAS IPs during ras recovery.
1989 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1990 &con->disable_ras_err_cnt_harvest);
1994 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
1995 struct ras_fs_if *head,
1998 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
2005 memcpy(obj->fs_data.debugfs_name,
2007 sizeof(obj->fs_data.debugfs_name));
2009 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
2010 obj, &amdgpu_ras_debugfs_ops);
2013 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev)
2017 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
2018 case IP_VERSION(13, 0, 6):
2019 case IP_VERSION(13, 0, 12):
2020 case IP_VERSION(13, 0, 14):
2031 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
2033 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2035 struct ras_manager *obj;
2036 struct ras_fs_if fs_info;
2039 * it won't be called in resume path, no need to check
2040 * suspend and gpu reset status
2042 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
2045 dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
2047 list_for_each_entry(obj, &con->head, node) {
2048 if (amdgpu_ras_is_supported(adev, obj->head.block) &&
2049 (obj->attr_inuse == 1)) {
2050 sprintf(fs_info.debugfs_name, "%s_err_inject",
2051 get_ras_block_str(&obj->head));
2052 fs_info.head = obj->head;
2053 amdgpu_ras_debugfs_create(adev, &fs_info, dir);
2057 if (amdgpu_ras_aca_is_supported(adev)) {
2058 if (amdgpu_aca_is_enabled(adev))
2059 amdgpu_aca_smu_debugfs_init(adev, dir);
2061 amdgpu_mca_smu_debugfs_init(adev, dir);
2068 static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
2069 amdgpu_ras_sysfs_badpages_read, NULL, 0);
2070 static DEVICE_ATTR(features, S_IRUGO,
2071 amdgpu_ras_sysfs_features_read, NULL);
2072 static DEVICE_ATTR(version, 0444,
2073 amdgpu_ras_sysfs_version_show, NULL);
2074 static DEVICE_ATTR(schema, 0444,
2075 amdgpu_ras_sysfs_schema_show, NULL);
2076 static DEVICE_ATTR(event_state, 0444,
2077 amdgpu_ras_sysfs_event_state_show, NULL);
2078 static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
2080 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2081 struct attribute_group group = {
2082 .name = RAS_FS_NAME,
2084 struct attribute *attrs[] = {
2085 &con->features_attr.attr,
2086 &con->version_attr.attr,
2087 &con->schema_attr.attr,
2088 &con->event_state_attr.attr,
2091 struct bin_attribute *bin_attrs[] = {
2097 group.attrs = attrs;
2099 /* add features entry */
2100 con->features_attr = dev_attr_features;
2101 sysfs_attr_init(attrs[0]);
2103 /* add version entry */
2104 con->version_attr = dev_attr_version;
2105 sysfs_attr_init(attrs[1]);
2107 /* add schema entry */
2108 con->schema_attr = dev_attr_schema;
2109 sysfs_attr_init(attrs[2]);
2111 /* add event_state entry */
2112 con->event_state_attr = dev_attr_event_state;
2113 sysfs_attr_init(attrs[3]);
2115 if (amdgpu_bad_page_threshold != 0) {
2116 /* add bad_page_features entry */
2117 bin_attr_gpu_vram_bad_pages.private = NULL;
2118 con->badpages_attr = bin_attr_gpu_vram_bad_pages;
2119 bin_attrs[0] = &con->badpages_attr;
2120 group.bin_attrs = bin_attrs;
2121 sysfs_bin_attr_init(bin_attrs[0]);
2124 r = sysfs_create_group(&adev->dev->kobj, &group);
2126 dev_err(adev->dev, "Failed to create RAS sysfs group!");
2131 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
2133 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2134 struct ras_manager *con_obj, *ip_obj, *tmp;
2136 if (IS_ENABLED(CONFIG_DEBUG_FS)) {
2137 list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
2138 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
2144 amdgpu_ras_sysfs_remove_all(adev);
2151 /* For the hardware that cannot enable bif ring for both ras_controller_irq
2152 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
2153 * register to check whether the interrupt is triggered or not, and properly
2154 * ack the interrupt if it is there
2156 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
2158 /* Fatal error events are handled on host side */
2159 if (amdgpu_sriov_vf(adev))
2162 * If the current interrupt is caused by a non-fatal RAS error, skip
2163 * check for fatal error. For fatal errors, FED status of all devices
2164 * in XGMI hive gets set when the first device gets fatal error
2165 * interrupt. The error gets propagated to other devices as well, so
2166 * make sure to ack the interrupt regardless of FED status.
2168 if (!amdgpu_ras_get_fed_status(adev) &&
2169 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY))
2172 if (adev->nbio.ras &&
2173 adev->nbio.ras->handle_ras_controller_intr_no_bifring)
2174 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
2176 if (adev->nbio.ras &&
2177 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
2178 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
2181 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
2182 struct amdgpu_iv_entry *entry)
2184 bool poison_stat = false;
2185 struct amdgpu_device *adev = obj->adev;
2186 struct amdgpu_ras_block_object *block_obj =
2187 amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
2188 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2189 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION;
2193 if (!block_obj || !con)
2196 ret = amdgpu_ras_mark_ras_event(adev, type);
2200 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block);
2201 /* both query_poison_status and handle_poison_consumption are optional,
2202 * but at least one of them should be implemented if we need poison
2203 * consumption handler
2205 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) {
2206 poison_stat = block_obj->hw_ops->query_poison_status(adev);
2208 /* Not poison consumption interrupt, no need to handle it */
2209 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
2210 block_obj->ras_comm.name);
2216 amdgpu_umc_poison_handler(adev, obj->head.block, 0);
2218 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption)
2219 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
2221 /* gpu reset is fallback for failed and default cases.
2222 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset.
2224 if (poison_stat && !amdgpu_ras_is_rma(adev)) {
2225 event_id = amdgpu_ras_acquire_event_id(adev, type);
2226 RAS_EVENT_LOG(adev, event_id,
2227 "GPU reset for %s RAS poison consumption is issued!\n",
2228 block_obj->ras_comm.name);
2229 amdgpu_ras_reset_gpu(adev);
2233 amdgpu_gfx_poison_consumption_handler(adev, entry);
2236 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
2237 struct amdgpu_iv_entry *entry)
2239 struct amdgpu_device *adev = obj->adev;
2240 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
2244 ret = amdgpu_ras_mark_ras_event(adev, type);
2248 event_id = amdgpu_ras_acquire_event_id(adev, type);
2249 RAS_EVENT_LOG(adev, event_id, "Poison is created\n");
2251 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) {
2252 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev);
2254 atomic_inc(&con->page_retirement_req_cnt);
2255 atomic_inc(&con->poison_creation_count);
2257 wake_up(&con->page_retirement_wq);
2261 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
2262 struct amdgpu_iv_entry *entry)
2264 struct ras_ih_data *data = &obj->ih_data;
2265 struct ras_err_data err_data;
2271 ret = amdgpu_ras_error_data_init(&err_data);
2275 /* Let IP handle its data, maybe we need get the output
2276 * from the callback to update the error type/count, etc
2278 amdgpu_ras_set_fed(obj->adev, true);
2279 ret = data->cb(obj->adev, &err_data, entry);
2280 /* ue will trigger an interrupt, and in that case
2281 * we need do a reset to recovery the whole system.
2282 * But leave IP do that recovery, here we just dispatch
2285 if (ret == AMDGPU_RAS_SUCCESS) {
2286 /* these counts could be left as 0 if
2287 * some blocks do not count error number
2289 obj->err_data.ue_count += err_data.ue_count;
2290 obj->err_data.ce_count += err_data.ce_count;
2291 obj->err_data.de_count += err_data.de_count;
2294 amdgpu_ras_error_data_fini(&err_data);
2297 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
2299 struct ras_ih_data *data = &obj->ih_data;
2300 struct amdgpu_iv_entry entry;
2302 while (data->rptr != data->wptr) {
2304 memcpy(&entry, &data->ring[data->rptr],
2305 data->element_size);
2308 data->rptr = (data->aligned_element_size +
2309 data->rptr) % data->ring_size;
2311 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
2312 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2313 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
2315 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
2317 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2318 amdgpu_ras_interrupt_umc_handler(obj, &entry);
2320 dev_warn(obj->adev->dev,
2321 "No RAS interrupt handler for non-UMC block with poison disabled.\n");
2326 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
2328 struct ras_ih_data *data =
2329 container_of(work, struct ras_ih_data, ih_work);
2330 struct ras_manager *obj =
2331 container_of(data, struct ras_manager, ih_data);
2333 amdgpu_ras_interrupt_handler(obj);
2336 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
2337 struct ras_dispatch_if *info)
2339 struct ras_manager *obj;
2340 struct ras_ih_data *data;
2342 obj = amdgpu_ras_find_obj(adev, &info->head);
2346 data = &obj->ih_data;
2348 if (data->inuse == 0)
2351 /* Might be overflow... */
2352 memcpy(&data->ring[data->wptr], info->entry,
2353 data->element_size);
2356 data->wptr = (data->aligned_element_size +
2357 data->wptr) % data->ring_size;
2359 schedule_work(&data->ih_work);
2364 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
2365 struct ras_common_if *head)
2367 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2368 struct ras_ih_data *data;
2373 data = &obj->ih_data;
2374 if (data->inuse == 0)
2377 cancel_work_sync(&data->ih_work);
2380 memset(data, 0, sizeof(*data));
2386 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
2387 struct ras_common_if *head)
2389 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2390 struct ras_ih_data *data;
2391 struct amdgpu_ras_block_object *ras_obj;
2394 /* in case we registe the IH before enable ras feature */
2395 obj = amdgpu_ras_create_obj(adev, head);
2401 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
2403 data = &obj->ih_data;
2404 /* add the callback.etc */
2405 *data = (struct ras_ih_data) {
2407 .cb = ras_obj->ras_cb,
2408 .element_size = sizeof(struct amdgpu_iv_entry),
2413 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
2415 data->aligned_element_size = ALIGN(data->element_size, 8);
2416 /* the ring can store 64 iv entries. */
2417 data->ring_size = 64 * data->aligned_element_size;
2418 data->ring = kmalloc(data->ring_size, GFP_KERNEL);
2430 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
2432 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2433 struct ras_manager *obj, *tmp;
2435 list_for_each_entry_safe(obj, tmp, &con->head, node) {
2436 amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
2443 /* traversal all IPs except NBIO to query error counter */
2444 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type)
2446 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2447 struct ras_manager *obj;
2449 if (!adev->ras_enabled || !con)
2452 list_for_each_entry(obj, &con->head, node) {
2453 struct ras_query_if info = {
2458 * PCIE_BIF IP has one different isr by ras controller
2459 * interrupt, the specific ras counter query will be
2460 * done in that isr. So skip such block from common
2461 * sync flood interrupt isr calling.
2463 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
2467 * this is a workaround for aldebaran, skip send msg to
2468 * smu to get ecc_info table due to smu handle get ecc
2469 * info table failed temporarily.
2470 * should be removed until smu fix handle ecc_info table.
2472 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
2473 (amdgpu_ip_version(adev, MP1_HWIP, 0) ==
2474 IP_VERSION(13, 0, 2)))
2477 amdgpu_ras_query_error_status_with_event(adev, &info, type);
2479 if (amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2480 IP_VERSION(11, 0, 2) &&
2481 amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2482 IP_VERSION(11, 0, 4) &&
2483 amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2484 IP_VERSION(13, 0, 0)) {
2485 if (amdgpu_ras_reset_error_status(adev, info.head.block))
2486 dev_warn(adev->dev, "Failed to reset error counter and error status");
2491 /* Parse RdRspStatus and WrRspStatus */
2492 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
2493 struct ras_query_if *info)
2495 struct amdgpu_ras_block_object *block_obj;
2497 * Only two block need to query read/write
2498 * RspStatus at current state
2500 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
2501 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
2504 block_obj = amdgpu_ras_get_ras_block(adev,
2506 info->head.sub_block_index);
2508 if (!block_obj || !block_obj->hw_ops) {
2509 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
2510 get_ras_block_str(&info->head));
2514 if (block_obj->hw_ops->query_ras_error_status)
2515 block_obj->hw_ops->query_ras_error_status(adev);
2519 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
2521 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2522 struct ras_manager *obj;
2524 if (!adev->ras_enabled || !con)
2527 list_for_each_entry(obj, &con->head, node) {
2528 struct ras_query_if info = {
2532 amdgpu_ras_error_status_query(adev, &info);
2536 /* recovery begin */
2538 /* return 0 on success.
2539 * caller need free bps.
2541 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
2542 struct ras_badpage **bps, unsigned int *count)
2544 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2545 struct ras_err_handler_data *data;
2547 int ret = 0, status;
2549 if (!con || !con->eh_data || !bps || !count)
2552 mutex_lock(&con->recovery_lock);
2553 data = con->eh_data;
2554 if (!data || data->count == 0) {
2560 *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
2566 for (; i < data->count; i++) {
2567 (*bps)[i] = (struct ras_badpage){
2568 .bp = data->bps[i].retired_page,
2569 .size = AMDGPU_GPU_PAGE_SIZE,
2570 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
2572 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
2573 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT);
2574 if (status == -EBUSY)
2575 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
2576 else if (status == -ENOENT)
2577 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
2580 *count = data->count;
2582 mutex_unlock(&con->recovery_lock);
2586 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev,
2587 struct amdgpu_hive_info *hive, bool status)
2589 struct amdgpu_device *tmp_adev;
2592 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
2593 amdgpu_ras_set_fed(tmp_adev, status);
2595 amdgpu_ras_set_fed(adev, status);
2599 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev)
2601 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2602 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2603 int hive_ras_recovery = 0;
2606 hive_ras_recovery = atomic_read(&hive->ras_recovery);
2607 amdgpu_put_xgmi_hive(hive);
2610 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery))
2616 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev)
2618 if (amdgpu_ras_intr_triggered())
2619 return RAS_EVENT_TYPE_FATAL;
2621 return RAS_EVENT_TYPE_POISON_CONSUMPTION;
2624 static void amdgpu_ras_do_recovery(struct work_struct *work)
2626 struct amdgpu_ras *ras =
2627 container_of(work, struct amdgpu_ras, recovery_work);
2628 struct amdgpu_device *remote_adev = NULL;
2629 struct amdgpu_device *adev = ras->adev;
2630 struct list_head device_list, *device_list_handle = NULL;
2631 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2632 enum ras_event_type type;
2635 atomic_set(&hive->ras_recovery, 1);
2637 /* If any device which is part of the hive received RAS fatal
2638 * error interrupt, set fatal error status on all. This
2639 * condition will need a recovery, and flag will be cleared
2640 * as part of recovery.
2642 list_for_each_entry(remote_adev, &hive->device_list,
2644 if (amdgpu_ras_get_fed_status(remote_adev)) {
2645 amdgpu_ras_set_fed_all(adev, hive, true);
2649 if (!ras->disable_ras_err_cnt_harvest) {
2651 /* Build list of devices to query RAS related errors */
2652 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
2653 device_list_handle = &hive->device_list;
2655 INIT_LIST_HEAD(&device_list);
2656 list_add_tail(&adev->gmc.xgmi.head, &device_list);
2657 device_list_handle = &device_list;
2660 type = amdgpu_ras_get_fatal_error_event(adev);
2661 list_for_each_entry(remote_adev,
2662 device_list_handle, gmc.xgmi.head) {
2663 amdgpu_ras_query_err_status(remote_adev);
2664 amdgpu_ras_log_on_err_counter(remote_adev, type);
2669 if (amdgpu_device_should_recover_gpu(ras->adev)) {
2670 struct amdgpu_reset_context reset_context;
2671 memset(&reset_context, 0, sizeof(reset_context));
2673 reset_context.method = AMD_RESET_METHOD_NONE;
2674 reset_context.reset_req_dev = adev;
2675 reset_context.src = AMDGPU_RESET_SRC_RAS;
2676 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
2678 /* Perform full reset in fatal error mode */
2679 if (!amdgpu_ras_is_poison_mode_supported(ras->adev))
2680 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2682 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2684 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) {
2685 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET;
2686 reset_context.method = AMD_RESET_METHOD_MODE2;
2689 /* Fatal error occurs in poison mode, mode1 reset is used to
2692 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) {
2693 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET;
2694 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2696 psp_fatal_error_recovery_quirk(&adev->psp);
2700 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
2702 atomic_set(&ras->in_recovery, 0);
2704 atomic_set(&hive->ras_recovery, 0);
2705 amdgpu_put_xgmi_hive(hive);
2709 /* alloc/realloc bps array */
2710 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
2711 struct ras_err_handler_data *data, int pages)
2713 unsigned int old_space = data->count + data->space_left;
2714 unsigned int new_space = old_space + pages;
2715 unsigned int align_space = ALIGN(new_space, 512);
2716 void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
2723 memcpy(bps, data->bps,
2724 data->count * sizeof(*data->bps));
2729 data->space_left += align_space - old_space;
2733 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev,
2734 struct eeprom_table_record *bps,
2735 struct ras_err_data *err_data)
2737 struct ta_ras_query_address_input addr_in;
2738 uint32_t socket = 0;
2741 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2742 socket = adev->smuio.funcs->get_socket_id(adev);
2744 /* reinit err_data */
2745 err_data->err_addr_cnt = 0;
2746 err_data->err_addr_len = adev->umc.retire_unit;
2748 memset(&addr_in, 0, sizeof(addr_in));
2749 addr_in.ma.err_addr = bps->address;
2750 addr_in.ma.socket_id = socket;
2751 addr_in.ma.ch_inst = bps->mem_channel;
2752 /* tell RAS TA the node instance is not used */
2753 addr_in.ma.node_inst = TA_RAS_INV_NODE;
2755 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2756 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data,
2757 &addr_in, NULL, false);
2762 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev,
2763 struct eeprom_table_record *bps,
2764 struct ras_err_data *err_data)
2766 struct ta_ras_query_address_input addr_in;
2767 uint32_t die_id, socket = 0;
2769 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2770 socket = adev->smuio.funcs->get_socket_id(adev);
2772 /* although die id is gotten from PA in nps1 mode, the id is
2773 * fitable for any nps mode
2775 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa)
2776 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address,
2777 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT);
2781 /* reinit err_data */
2782 err_data->err_addr_cnt = 0;
2783 err_data->err_addr_len = adev->umc.retire_unit;
2785 memset(&addr_in, 0, sizeof(addr_in));
2786 addr_in.ma.err_addr = bps->address;
2787 addr_in.ma.ch_inst = bps->mem_channel;
2788 addr_in.ma.umc_inst = bps->mcumc_id;
2789 addr_in.ma.node_inst = die_id;
2790 addr_in.ma.socket_id = socket;
2792 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2793 return adev->umc.ras->convert_ras_err_addr(adev, err_data,
2794 &addr_in, NULL, false);
2799 /* it deal with vram only. */
2800 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
2801 struct eeprom_table_record *bps, int pages, bool from_rom)
2803 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2804 struct ras_err_handler_data *data;
2805 struct ras_err_data err_data;
2806 struct eeprom_table_record *err_rec;
2807 struct amdgpu_ras_eeprom_control *control =
2808 &adev->psp.ras_context.ras->eeprom_control;
2809 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE;
2811 uint32_t i, j, loop_cnt = 1;
2812 bool find_pages_per_pa = false;
2814 if (!con || !con->eh_data || !bps || pages <= 0)
2819 kcalloc(adev->umc.retire_unit,
2820 sizeof(struct eeprom_table_record), GFP_KERNEL);
2821 if (!err_data.err_addr) {
2822 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n");
2827 err_rec = err_data.err_addr;
2828 loop_cnt = adev->umc.retire_unit;
2829 if (adev->gmc.gmc_funcs->query_mem_partition_mode)
2830 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
2833 mutex_lock(&con->recovery_lock);
2834 data = con->eh_data;
2836 /* Returning 0 as the absence of eh_data is acceptable */
2840 for (i = 0; i < pages; i++) {
2842 control->rec_type == AMDGPU_RAS_EEPROM_REC_MCA) {
2843 if (!find_pages_per_pa) {
2844 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[i], &err_data)) {
2845 if (!i && nps == AMDGPU_NPS1_PARTITION_MODE) {
2846 /* may use old RAS TA, use PA to find pages in
2849 if (amdgpu_umc_pages_in_a_row(adev, &err_data,
2850 bps[i].retired_page <<
2851 AMDGPU_GPU_PAGE_SHIFT)) {
2855 find_pages_per_pa = true;
2858 /* unsupported cases */
2864 if (amdgpu_umc_pages_in_a_row(adev, &err_data,
2865 bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT)) {
2871 if (from_rom && !find_pages_per_pa) {
2872 if (bps[i].retired_page & UMC_CHANNEL_IDX_V2) {
2873 /* bad page in any NPS mode in eeprom */
2874 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[i], &err_data)) {
2879 /* legacy bad page in eeprom, generated only in
2882 if (amdgpu_ras_mca2pa(adev, &bps[i], &err_data)) {
2883 /* old RAS TA or ASICs which don't support to
2884 * convert addrss via mca address
2886 if (!i && nps == AMDGPU_NPS1_PARTITION_MODE) {
2887 find_pages_per_pa = true;
2891 /* non-nps1 mode, old RAS TA
2900 if (!find_pages_per_pa)
2901 i += (adev->umc.retire_unit - 1);
2907 for (j = 0; j < loop_cnt; j++) {
2908 if (amdgpu_ras_check_bad_page_unlock(con,
2909 err_rec[j].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2912 if (!data->space_left &&
2913 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
2918 amdgpu_ras_reserve_page(adev, err_rec[j].retired_page);
2920 memcpy(&data->bps[data->count], &(err_rec[j]),
2921 sizeof(struct eeprom_table_record));
2929 kfree(err_data.err_addr);
2931 mutex_unlock(&con->recovery_lock);
2937 * write error record array to eeprom, the function should be
2938 * protected by recovery_lock
2939 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL
2941 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
2942 unsigned long *new_cnt)
2944 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2945 struct ras_err_handler_data *data;
2946 struct amdgpu_ras_eeprom_control *control;
2947 int save_count, unit_num, bad_page_num, i;
2949 if (!con || !con->eh_data) {
2956 mutex_lock(&con->recovery_lock);
2957 control = &con->eeprom_control;
2958 data = con->eh_data;
2959 bad_page_num = control->ras_num_bad_pages;
2960 save_count = data->count - bad_page_num;
2961 mutex_unlock(&con->recovery_lock);
2963 unit_num = save_count / adev->umc.retire_unit;
2965 *new_cnt = unit_num;
2967 /* only new entries are saved */
2968 if (save_count > 0) {
2969 if (control->rec_type == AMDGPU_RAS_EEPROM_REC_PA) {
2970 if (amdgpu_ras_eeprom_append(control,
2971 &data->bps[control->ras_num_recs],
2973 dev_err(adev->dev, "Failed to save EEPROM table data!");
2977 for (i = 0; i < unit_num; i++) {
2978 if (amdgpu_ras_eeprom_append(control,
2979 &data->bps[bad_page_num + i * adev->umc.retire_unit],
2981 dev_err(adev->dev, "Failed to save EEPROM table data!");
2987 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
2994 * read error record array in eeprom and reserve enough space for
2995 * storing new bad pages
2997 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
2999 struct amdgpu_ras_eeprom_control *control =
3000 &adev->psp.ras_context.ras->eeprom_control;
3001 struct eeprom_table_record *bps;
3004 /* no bad page record, skip eeprom access */
3005 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
3008 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
3012 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
3014 dev_err(adev->dev, "Failed to load EEPROM table records!");
3016 if (control->ras_num_recs > 1 &&
3017 adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {
3018 if ((bps[0].address == bps[1].address) &&
3019 (bps[0].mem_channel == bps[1].mem_channel))
3020 control->rec_type = AMDGPU_RAS_EEPROM_REC_PA;
3022 control->rec_type = AMDGPU_RAS_EEPROM_REC_MCA;
3025 ret = amdgpu_ras_eeprom_check(control);
3030 if (amdgpu_ras_is_rma(adev)) {
3035 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true);
3043 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
3046 struct ras_err_handler_data *data = con->eh_data;
3049 addr >>= AMDGPU_GPU_PAGE_SHIFT;
3050 for (i = 0; i < data->count; i++)
3051 if (addr == data->bps[i].retired_page)
3058 * check if an address belongs to bad page
3060 * Note: this check is only for umc block
3062 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
3065 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3068 if (!con || !con->eh_data)
3071 mutex_lock(&con->recovery_lock);
3072 ret = amdgpu_ras_check_bad_page_unlock(con, addr);
3073 mutex_unlock(&con->recovery_lock);
3077 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
3080 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3083 * Justification of value bad_page_cnt_threshold in ras structure
3085 * Generally, 0 <= amdgpu_bad_page_threshold <= max record length
3086 * in eeprom or amdgpu_bad_page_threshold == -2, introduce two
3087 * scenarios accordingly.
3089 * Bad page retirement enablement:
3090 * - If amdgpu_bad_page_threshold = -2,
3091 * bad_page_cnt_threshold = typical value by formula.
3093 * - When the value from user is 0 < amdgpu_bad_page_threshold <
3094 * max record length in eeprom, use it directly.
3096 * Bad page retirement disablement:
3097 * - If amdgpu_bad_page_threshold = 0, bad page retirement
3098 * functionality is disabled, and bad_page_cnt_threshold will
3102 if (amdgpu_bad_page_threshold < 0) {
3103 u64 val = adev->gmc.mc_vram_size;
3105 do_div(val, RAS_BAD_PAGE_COVER);
3106 con->bad_page_cnt_threshold = min(lower_32_bits(val),
3109 con->bad_page_cnt_threshold = min_t(int, max_count,
3110 amdgpu_bad_page_threshold);
3114 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
3115 enum amdgpu_ras_block block, uint16_t pasid,
3116 pasid_notify pasid_fn, void *data, uint32_t reset)
3119 struct ras_poison_msg poison_msg;
3120 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3122 memset(&poison_msg, 0, sizeof(poison_msg));
3123 poison_msg.block = block;
3124 poison_msg.pasid = pasid;
3125 poison_msg.reset = reset;
3126 poison_msg.pasid_fn = pasid_fn;
3127 poison_msg.data = data;
3129 ret = kfifo_put(&con->poison_fifo, poison_msg);
3131 dev_err(adev->dev, "Poison message fifo is full!\n");
3138 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev,
3139 struct ras_poison_msg *poison_msg)
3141 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3143 return kfifo_get(&con->poison_fifo, poison_msg);
3146 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log)
3148 mutex_init(&ecc_log->lock);
3150 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL);
3151 ecc_log->de_queried_count = 0;
3152 ecc_log->prev_de_queried_count = 0;
3155 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log)
3157 struct radix_tree_iter iter;
3159 struct ras_ecc_err *ecc_err;
3161 mutex_lock(&ecc_log->lock);
3162 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) {
3163 ecc_err = radix_tree_deref_slot(slot);
3164 kfree(ecc_err->err_pages.pfn);
3166 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot);
3168 mutex_unlock(&ecc_log->lock);
3170 mutex_destroy(&ecc_log->lock);
3171 ecc_log->de_queried_count = 0;
3172 ecc_log->prev_de_queried_count = 0;
3175 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con,
3176 uint32_t delayed_ms)
3180 mutex_lock(&con->umc_ecc_log.lock);
3181 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree,
3182 UMC_ECC_NEW_DETECTED_TAG);
3183 mutex_unlock(&con->umc_ecc_log.lock);
3186 schedule_delayed_work(&con->page_retirement_dwork,
3187 msecs_to_jiffies(delayed_ms));
3189 return ret ? true : false;
3192 static void amdgpu_ras_do_page_retirement(struct work_struct *work)
3194 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3195 page_retirement_dwork.work);
3196 struct amdgpu_device *adev = con->adev;
3197 struct ras_err_data err_data;
3198 unsigned long err_cnt;
3200 /* If gpu reset is ongoing, delay retiring the bad pages */
3201 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) {
3202 amdgpu_ras_schedule_retirement_dwork(con,
3203 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3);
3207 amdgpu_ras_error_data_init(&err_data);
3209 amdgpu_umc_handle_bad_pages(adev, &err_data);
3210 err_cnt = err_data.err_addr_cnt;
3212 amdgpu_ras_error_data_fini(&err_data);
3214 if (err_cnt && amdgpu_ras_is_rma(adev))
3215 amdgpu_ras_reset_gpu(adev);
3217 amdgpu_ras_schedule_retirement_dwork(con,
3218 AMDGPU_RAS_RETIRE_PAGE_INTERVAL);
3221 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev,
3222 uint32_t poison_creation_count)
3225 struct ras_ecc_log_info *ecc_log;
3226 struct ras_query_if info;
3227 uint32_t timeout = 0;
3228 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3229 uint64_t de_queried_count;
3230 uint32_t new_detect_count, total_detect_count;
3231 uint32_t need_query_count = poison_creation_count;
3232 bool query_data_timeout = false;
3233 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
3235 memset(&info, 0, sizeof(info));
3236 info.head.block = AMDGPU_RAS_BLOCK__UMC;
3238 ecc_log = &ras->umc_ecc_log;
3239 total_detect_count = 0;
3241 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type);
3245 de_queried_count = ecc_log->de_queried_count;
3246 if (de_queried_count > ecc_log->prev_de_queried_count) {
3247 new_detect_count = de_queried_count - ecc_log->prev_de_queried_count;
3248 ecc_log->prev_de_queried_count = de_queried_count;
3251 new_detect_count = 0;
3254 if (new_detect_count) {
3255 total_detect_count += new_detect_count;
3257 if (!timeout && need_query_count)
3258 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC;
3262 query_data_timeout = true;
3268 } while (total_detect_count < need_query_count);
3270 if (query_data_timeout) {
3271 dev_warn(adev->dev, "Can't find deferred error! count: %u\n",
3272 (need_query_count - total_detect_count));
3276 if (total_detect_count)
3277 schedule_delayed_work(&ras->page_retirement_dwork, 0);
3282 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev)
3284 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3285 struct ras_poison_msg msg;
3289 ret = kfifo_get(&con->poison_fifo, &msg);
3293 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev,
3294 uint32_t msg_count, uint32_t *gpu_reset)
3296 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3297 uint32_t reset_flags = 0, reset = 0;
3298 struct ras_poison_msg msg;
3301 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
3303 for (i = 0; i < msg_count; i++) {
3304 ret = amdgpu_ras_get_poison_req(adev, &msg);
3309 msg.pasid_fn(adev, msg.pasid, msg.data);
3311 reset_flags |= msg.reset;
3314 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */
3315 if (reset_flags && !amdgpu_ras_is_rma(adev)) {
3316 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET)
3317 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET;
3318 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET)
3319 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
3321 reset = reset_flags;
3323 flush_delayed_work(&con->page_retirement_dwork);
3325 con->gpu_reset_flags |= reset;
3326 amdgpu_ras_reset_gpu(adev);
3330 /* Wait for gpu recovery to complete */
3331 flush_work(&con->recovery_work);
3337 static int amdgpu_ras_page_retirement_thread(void *param)
3339 struct amdgpu_device *adev = (struct amdgpu_device *)param;
3340 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3341 uint32_t poison_creation_count, msg_count;
3345 while (!kthread_should_stop()) {
3347 wait_event_interruptible(con->page_retirement_wq,
3348 kthread_should_stop() ||
3349 atomic_read(&con->page_retirement_req_cnt));
3351 if (kthread_should_stop())
3357 poison_creation_count = atomic_read(&con->poison_creation_count);
3358 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count);
3362 if (poison_creation_count) {
3363 atomic_sub(poison_creation_count, &con->poison_creation_count);
3364 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt);
3366 } while (atomic_read(&con->poison_creation_count));
3369 msg_count = kfifo_len(&con->poison_fifo);
3371 ret = amdgpu_ras_poison_consumption_handler(adev,
3372 msg_count, &gpu_reset);
3373 if ((ret != -EIO) &&
3374 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET))
3375 atomic_sub(msg_count, &con->page_retirement_req_cnt);
3379 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) {
3380 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */
3381 /* Clear poison creation request */
3382 atomic_set(&con->poison_creation_count, 0);
3384 /* Clear poison fifo */
3385 amdgpu_ras_clear_poison_fifo(adev);
3387 /* Clear all poison requests */
3388 atomic_set(&con->page_retirement_req_cnt, 0);
3391 /* Wait for mode-1 reset to complete */
3392 down_read(&adev->reset_domain->sem);
3393 up_read(&adev->reset_domain->sem);
3396 /* Wake up work to save bad pages to eeprom */
3397 schedule_delayed_work(&con->page_retirement_dwork, 0);
3398 } else if (gpu_reset) {
3399 /* gpu just completed mode-2 reset or other reset */
3400 /* Clear poison consumption messages cached in fifo */
3401 msg_count = kfifo_len(&con->poison_fifo);
3403 amdgpu_ras_clear_poison_fifo(adev);
3404 atomic_sub(msg_count, &con->page_retirement_req_cnt);
3407 /* Wake up work to save bad pages to eeprom */
3408 schedule_delayed_work(&con->page_retirement_dwork, 0);
3415 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev)
3417 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3418 struct amdgpu_ras_eeprom_control *control;
3421 if (!con || amdgpu_sriov_vf(adev))
3424 control = &con->eeprom_control;
3425 ret = amdgpu_ras_eeprom_init(control);
3429 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr)
3430 control->rec_type = AMDGPU_RAS_EEPROM_REC_PA;
3432 /* default status is MCA storage */
3433 if (control->ras_num_recs <= 1 &&
3434 adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
3435 control->rec_type = AMDGPU_RAS_EEPROM_REC_MCA;
3437 if (control->ras_num_recs) {
3438 ret = amdgpu_ras_load_bad_pages(adev);
3442 amdgpu_dpm_send_hbm_bad_pages_num(
3443 adev, control->ras_num_bad_pages);
3445 if (con->update_channel_flag == true) {
3446 amdgpu_dpm_send_hbm_bad_channel_flag(
3447 adev, control->bad_channel_bitmap);
3448 con->update_channel_flag = false;
3455 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info)
3457 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3458 struct ras_err_handler_data **data;
3459 u32 max_eeprom_records_count = 0;
3462 if (!con || amdgpu_sriov_vf(adev))
3465 /* Allow access to RAS EEPROM via debugfs, when the ASIC
3466 * supports RAS and debugfs is enabled, but when
3467 * adev->ras_enabled is unset, i.e. when "ras_enable"
3468 * module parameter is set to 0.
3472 if (!adev->ras_enabled)
3475 data = &con->eh_data;
3476 *data = kzalloc(sizeof(**data), GFP_KERNEL);
3482 mutex_init(&con->recovery_lock);
3483 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
3484 atomic_set(&con->in_recovery, 0);
3485 con->eeprom_control.bad_channel_bitmap = 0;
3487 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control);
3488 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
3491 ret = amdgpu_ras_init_badpage_info(adev);
3496 mutex_init(&con->page_rsv_lock);
3497 INIT_KFIFO(con->poison_fifo);
3498 mutex_init(&con->page_retirement_lock);
3499 init_waitqueue_head(&con->page_retirement_wq);
3500 atomic_set(&con->page_retirement_req_cnt, 0);
3501 atomic_set(&con->poison_creation_count, 0);
3502 con->page_retirement_thread =
3503 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement");
3504 if (IS_ERR(con->page_retirement_thread)) {
3505 con->page_retirement_thread = NULL;
3506 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n");
3509 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement);
3510 amdgpu_ras_ecc_log_init(&con->umc_ecc_log);
3511 #ifdef CONFIG_X86_MCE_AMD
3512 if ((adev->asic_type == CHIP_ALDEBARAN) &&
3513 (adev->gmc.xgmi.connected_to_cpu))
3514 amdgpu_register_bad_pages_mca_notifier(adev);
3519 kfree((*data)->bps);
3521 con->eh_data = NULL;
3523 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
3526 * Except error threshold exceeding case, other failure cases in this
3527 * function would not fail amdgpu driver init.
3529 if (!amdgpu_ras_is_rma(adev))
3537 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
3539 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3540 struct ras_err_handler_data *data = con->eh_data;
3541 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES;
3544 /* recovery_init failed to init it, fini is useless */
3548 /* Save all cached bad pages to eeprom */
3550 flush_delayed_work(&con->page_retirement_dwork);
3551 ret = amdgpu_ras_schedule_retirement_dwork(con, 0);
3552 } while (ret && max_flush_timeout--);
3554 if (con->page_retirement_thread)
3555 kthread_stop(con->page_retirement_thread);
3557 atomic_set(&con->page_retirement_req_cnt, 0);
3558 atomic_set(&con->poison_creation_count, 0);
3560 mutex_destroy(&con->page_rsv_lock);
3562 cancel_work_sync(&con->recovery_work);
3564 cancel_delayed_work_sync(&con->page_retirement_dwork);
3566 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log);
3568 mutex_lock(&con->recovery_lock);
3569 con->eh_data = NULL;
3572 mutex_unlock(&con->recovery_lock);
3578 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
3580 if (amdgpu_sriov_vf(adev)) {
3581 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3582 case IP_VERSION(13, 0, 2):
3583 case IP_VERSION(13, 0, 6):
3584 case IP_VERSION(13, 0, 12):
3585 case IP_VERSION(13, 0, 14):
3592 if (adev->asic_type == CHIP_IP_DISCOVERY) {
3593 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3594 case IP_VERSION(13, 0, 0):
3595 case IP_VERSION(13, 0, 6):
3596 case IP_VERSION(13, 0, 10):
3597 case IP_VERSION(13, 0, 12):
3598 case IP_VERSION(13, 0, 14):
3599 case IP_VERSION(14, 0, 3):
3606 return adev->asic_type == CHIP_VEGA10 ||
3607 adev->asic_type == CHIP_VEGA20 ||
3608 adev->asic_type == CHIP_ARCTURUS ||
3609 adev->asic_type == CHIP_ALDEBARAN ||
3610 adev->asic_type == CHIP_SIENNA_CICHLID;
3614 * this is workaround for vega20 workstation sku,
3615 * force enable gfx ras, ignore vbios gfx ras flag
3616 * due to GC EDC can not write
3618 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
3620 struct atom_context *ctx = adev->mode_info.atom_context;
3625 if (strnstr(ctx->vbios_pn, "D16406",
3626 sizeof(ctx->vbios_pn)) ||
3627 strnstr(ctx->vbios_pn, "D36002",
3628 sizeof(ctx->vbios_pn)))
3629 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
3632 /* Query ras capablity via atomfirmware interface */
3633 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev)
3636 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
3637 dev_info(adev->dev, "MEM ECC is active.\n");
3638 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
3639 1 << AMDGPU_RAS_BLOCK__DF);
3641 dev_info(adev->dev, "MEM ECC is not presented.\n");
3645 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
3646 dev_info(adev->dev, "SRAM ECC is active.\n");
3647 if (!amdgpu_sriov_vf(adev))
3648 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
3649 1 << AMDGPU_RAS_BLOCK__DF);
3651 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
3652 1 << AMDGPU_RAS_BLOCK__SDMA |
3653 1 << AMDGPU_RAS_BLOCK__GFX);
3656 * VCN/JPEG RAS can be supported on both bare metal and
3659 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) ||
3660 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) ||
3661 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3))
3662 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
3663 1 << AMDGPU_RAS_BLOCK__JPEG);
3665 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
3666 1 << AMDGPU_RAS_BLOCK__JPEG);
3669 * XGMI RAS is not supported if xgmi num physical nodes
3672 if (!adev->gmc.xgmi.num_physical_nodes)
3673 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL);
3675 dev_info(adev->dev, "SRAM ECC is not presented.\n");
3679 /* Query poison mode from umc/df IP callbacks */
3680 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev)
3682 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3683 bool df_poison, umc_poison;
3685 /* poison setting is useless on SRIOV guest */
3686 if (amdgpu_sriov_vf(adev) || !con)
3689 /* Init poison supported flag, the default value is false */
3690 if (adev->gmc.xgmi.connected_to_cpu ||
3691 adev->gmc.is_app_apu) {
3692 /* enabled by default when GPU is connected to CPU */
3693 con->poison_supported = true;
3694 } else if (adev->df.funcs &&
3695 adev->df.funcs->query_ras_poison_mode &&
3697 adev->umc.ras->query_ras_poison_mode) {
3699 adev->df.funcs->query_ras_poison_mode(adev);
3701 adev->umc.ras->query_ras_poison_mode(adev);
3703 /* Only poison is set in both DF and UMC, we can support it */
3704 if (df_poison && umc_poison)
3705 con->poison_supported = true;
3706 else if (df_poison != umc_poison)
3708 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
3709 df_poison, umc_poison);
3714 * check hardware's ras ability which will be saved in hw_supported.
3715 * if hardware does not support ras, we can skip some ras initializtion and
3716 * forbid some ras operations from IP.
3717 * if software itself, say boot parameter, limit the ras ability. We still
3718 * need allow IP do some limited operations, like disable. In such case,
3719 * we have to initialize ras as normal. but need check if operation is
3720 * allowed or not in each function.
3722 static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
3724 adev->ras_hw_enabled = adev->ras_enabled = 0;
3726 if (!amdgpu_ras_asic_supported(adev))
3729 if (amdgpu_sriov_vf(adev)) {
3730 if (amdgpu_virt_get_ras_capability(adev))
3731 goto init_ras_enabled_flag;
3734 /* query ras capability from psp */
3735 if (amdgpu_psp_get_ras_capability(&adev->psp))
3736 goto init_ras_enabled_flag;
3738 /* query ras capablity from bios */
3739 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
3740 amdgpu_ras_query_ras_capablity_from_vbios(adev);
3742 /* driver only manages a few IP blocks RAS feature
3743 * when GPU is connected cpu through XGMI */
3744 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
3745 1 << AMDGPU_RAS_BLOCK__SDMA |
3746 1 << AMDGPU_RAS_BLOCK__MMHUB);
3749 /* apply asic specific settings (vega20 only for now) */
3750 amdgpu_ras_get_quirks(adev);
3752 /* query poison mode from umc/df ip callback */
3753 amdgpu_ras_query_poison_mode(adev);
3755 init_ras_enabled_flag:
3756 /* hw_supported needs to be aligned with RAS block mask. */
3757 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
3759 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
3760 adev->ras_hw_enabled & amdgpu_ras_mask;
3762 /* aca is disabled by default */
3763 adev->aca.is_enabled = false;
3765 /* bad page feature is not applicable to specific app platform */
3766 if (adev->gmc.is_app_apu &&
3767 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0))
3768 amdgpu_bad_page_threshold = 0;
3771 static void amdgpu_ras_counte_dw(struct work_struct *work)
3773 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3774 ras_counte_delay_work.work);
3775 struct amdgpu_device *adev = con->adev;
3776 struct drm_device *dev = adev_to_drm(adev);
3777 unsigned long ce_count, ue_count;
3780 res = pm_runtime_get_sync(dev->dev);
3784 /* Cache new values.
3786 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) {
3787 atomic_set(&con->ras_ce_count, ce_count);
3788 atomic_set(&con->ras_ue_count, ue_count);
3791 pm_runtime_mark_last_busy(dev->dev);
3793 pm_runtime_put_autosuspend(dev->dev);
3796 static int amdgpu_get_ras_schema(struct amdgpu_device *adev)
3798 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 |
3799 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE |
3800 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE |
3801 AMDGPU_RAS_ERROR__PARITY;
3804 static void ras_event_mgr_init(struct ras_event_manager *mgr)
3806 struct ras_event_state *event_state;
3809 memset(mgr, 0, sizeof(*mgr));
3810 atomic64_set(&mgr->seqno, 0);
3812 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) {
3813 event_state = &mgr->event_state[i];
3814 event_state->last_seqno = RAS_EVENT_INVALID_ID;
3815 atomic64_set(&event_state->count, 0);
3819 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev)
3821 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3822 struct amdgpu_hive_info *hive;
3827 hive = amdgpu_get_xgmi_hive(adev);
3828 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr;
3830 /* init event manager with node 0 on xgmi system */
3831 if (!amdgpu_reset_in_recovery(adev)) {
3832 if (!hive || adev->gmc.xgmi.node_id == 0)
3833 ras_event_mgr_init(ras->event_mgr);
3837 amdgpu_put_xgmi_hive(hive);
3840 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev)
3842 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3844 if (!con || (adev->flags & AMD_IS_APU))
3847 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3848 case IP_VERSION(13, 0, 2):
3849 case IP_VERSION(13, 0, 6):
3850 case IP_VERSION(13, 0, 12):
3851 case IP_VERSION(13, 0, 14):
3852 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE;
3859 int amdgpu_ras_init(struct amdgpu_device *adev)
3861 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3867 con = kzalloc(sizeof(*con) +
3868 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
3869 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
3875 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
3876 atomic_set(&con->ras_ce_count, 0);
3877 atomic_set(&con->ras_ue_count, 0);
3879 con->objs = (struct ras_manager *)(con + 1);
3881 amdgpu_ras_set_context(adev, con);
3883 amdgpu_ras_check_supported(adev);
3885 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
3886 /* set gfx block ras context feature for VEGA20 Gaming
3887 * send ras disable cmd to ras ta during ras late init.
3889 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
3890 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
3899 con->update_channel_flag = false;
3902 INIT_LIST_HEAD(&con->head);
3903 /* Might need get this flag from vbios. */
3904 con->flags = RAS_DEFAULT_FLAGS;
3906 /* initialize nbio ras function ahead of any other
3907 * ras functions so hardware fatal error interrupt
3908 * can be enabled as early as possible */
3909 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) {
3910 case IP_VERSION(7, 4, 0):
3911 case IP_VERSION(7, 4, 1):
3912 case IP_VERSION(7, 4, 4):
3913 if (!adev->gmc.xgmi.connected_to_cpu)
3914 adev->nbio.ras = &nbio_v7_4_ras;
3916 case IP_VERSION(4, 3, 0):
3917 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
3918 /* unlike other generation of nbio ras,
3919 * nbio v4_3 only support fatal error interrupt
3920 * to inform software that DF is freezed due to
3921 * system fatal error event. driver should not
3922 * enable nbio ras in such case. Instead,
3924 adev->nbio.ras = &nbio_v4_3_ras;
3926 case IP_VERSION(6, 3, 1):
3927 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
3928 /* unlike other generation of nbio ras,
3929 * nbif v6_3_1 only support fatal error interrupt
3930 * to inform software that DF is freezed due to
3931 * system fatal error event. driver should not
3932 * enable nbio ras in such case. Instead,
3935 adev->nbio.ras = &nbif_v6_3_1_ras;
3937 case IP_VERSION(7, 9, 0):
3938 case IP_VERSION(7, 9, 1):
3939 if (!adev->gmc.is_app_apu)
3940 adev->nbio.ras = &nbio_v7_9_ras;
3943 /* nbio ras is not available */
3947 /* nbio ras block needs to be enabled ahead of other ras blocks
3948 * to handle fatal error */
3949 r = amdgpu_nbio_ras_sw_init(adev);
3953 if (adev->nbio.ras &&
3954 adev->nbio.ras->init_ras_controller_interrupt) {
3955 r = adev->nbio.ras->init_ras_controller_interrupt(adev);
3960 if (adev->nbio.ras &&
3961 adev->nbio.ras->init_ras_err_event_athub_interrupt) {
3962 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
3967 /* Packed socket_id to ras feature mask bits[31:29] */
3968 if (adev->smuio.funcs &&
3969 adev->smuio.funcs->get_socket_id)
3970 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) <<
3971 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT);
3973 /* Get RAS schema for particular SOC */
3974 con->schema = amdgpu_get_ras_schema(adev);
3976 amdgpu_ras_init_reserved_vram_size(adev);
3978 if (amdgpu_ras_fs_init(adev)) {
3983 if (amdgpu_ras_aca_is_supported(adev)) {
3984 if (amdgpu_aca_is_enabled(adev))
3985 r = amdgpu_aca_init(adev);
3987 r = amdgpu_mca_init(adev);
3992 dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
3993 "hardware ability[%x] ras_mask[%x]\n",
3994 adev->ras_hw_enabled, adev->ras_enabled);
3998 amdgpu_ras_set_context(adev, NULL);
4004 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
4006 if (adev->gmc.xgmi.connected_to_cpu ||
4007 adev->gmc.is_app_apu)
4012 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
4013 struct ras_common_if *ras_block)
4015 struct ras_query_if info = {
4019 if (!amdgpu_persistent_edc_harvesting_supported(adev))
4022 if (amdgpu_ras_query_error_status(adev, &info) != 0)
4023 DRM_WARN("RAS init harvest failure");
4025 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
4026 DRM_WARN("RAS init harvest reset failure");
4031 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
4033 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4038 return con->poison_supported;
4041 /* helper function to handle common stuff in ip late init phase */
4042 int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
4043 struct ras_common_if *ras_block)
4045 struct amdgpu_ras_block_object *ras_obj = NULL;
4046 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4047 struct ras_query_if *query_info;
4048 unsigned long ue_count, ce_count;
4051 /* disable RAS feature per IP block if it is not supported */
4052 if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
4053 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
4057 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
4059 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) {
4060 /* in resume phase, if fail to enable ras,
4061 * clean up all ras fs nodes, and disable ras */
4067 /* check for errors on warm reset edc persisant supported ASIC */
4068 amdgpu_persistent_edc_harvesting(adev, ras_block);
4070 /* in resume phase, no need to create ras fs node */
4071 if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
4074 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4075 if (ras_obj->ras_cb || (ras_obj->hw_ops &&
4076 (ras_obj->hw_ops->query_poison_status ||
4077 ras_obj->hw_ops->handle_poison_consumption))) {
4078 r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
4083 if (ras_obj->hw_ops &&
4084 (ras_obj->hw_ops->query_ras_error_count ||
4085 ras_obj->hw_ops->query_ras_error_status)) {
4086 r = amdgpu_ras_sysfs_create(adev, ras_block);
4090 /* Those are the cached values at init.
4092 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL);
4095 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if));
4097 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) {
4098 atomic_set(&con->ras_ce_count, ce_count);
4099 atomic_set(&con->ras_ue_count, ue_count);
4108 if (ras_obj->ras_cb)
4109 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
4111 amdgpu_ras_feature_enable(adev, ras_block, 0);
4115 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
4116 struct ras_common_if *ras_block)
4118 return amdgpu_ras_block_late_init(adev, ras_block);
4121 /* helper function to remove ras fs node and interrupt handler */
4122 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
4123 struct ras_common_if *ras_block)
4125 struct amdgpu_ras_block_object *ras_obj;
4129 amdgpu_ras_sysfs_remove(adev, ras_block);
4131 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4132 if (ras_obj->ras_cb)
4133 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
4136 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
4137 struct ras_common_if *ras_block)
4139 return amdgpu_ras_block_late_fini(adev, ras_block);
4142 /* do some init work after IP late init as dependence.
4143 * and it runs in resume/gpu reset/booting up cases.
4145 void amdgpu_ras_resume(struct amdgpu_device *adev)
4147 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4148 struct ras_manager *obj, *tmp;
4150 if (!adev->ras_enabled || !con) {
4151 /* clean ras context for VEGA20 Gaming after send ras disable cmd */
4152 amdgpu_release_ras_context(adev);
4157 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
4158 /* Set up all other IPs which are not implemented. There is a
4159 * tricky thing that IP's actual ras error type should be
4160 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
4161 * ERROR_NONE make sense anyway.
4163 amdgpu_ras_enable_all_features(adev, 1);
4165 /* We enable ras on all hw_supported block, but as boot
4166 * parameter might disable some of them and one or more IP has
4167 * not implemented yet. So we disable them on behalf.
4169 list_for_each_entry_safe(obj, tmp, &con->head, node) {
4170 if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
4171 amdgpu_ras_feature_enable(adev, &obj->head, 0);
4172 /* there should be no any reference. */
4173 WARN_ON(alive_obj(obj));
4179 void amdgpu_ras_suspend(struct amdgpu_device *adev)
4181 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4183 if (!adev->ras_enabled || !con)
4186 amdgpu_ras_disable_all_features(adev, 0);
4187 /* Make sure all ras objects are disabled. */
4188 if (AMDGPU_RAS_GET_FEATURES(con->features))
4189 amdgpu_ras_disable_all_features(adev, 1);
4192 int amdgpu_ras_late_init(struct amdgpu_device *adev)
4194 struct amdgpu_ras_block_list *node, *tmp;
4195 struct amdgpu_ras_block_object *obj;
4198 amdgpu_ras_event_mgr_init(adev);
4200 if (amdgpu_ras_aca_is_supported(adev)) {
4201 if (amdgpu_reset_in_recovery(adev)) {
4202 if (amdgpu_aca_is_enabled(adev))
4203 r = amdgpu_aca_reset(adev);
4205 r = amdgpu_mca_reset(adev);
4210 if (!amdgpu_sriov_vf(adev)) {
4211 if (amdgpu_aca_is_enabled(adev))
4212 amdgpu_ras_set_aca_debug_mode(adev, false);
4214 amdgpu_ras_set_mca_debug_mode(adev, false);
4218 /* Guest side doesn't need init ras feature */
4219 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev))
4222 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
4223 obj = node->ras_obj;
4225 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
4229 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block))
4232 if (obj->ras_late_init) {
4233 r = obj->ras_late_init(adev, &obj->ras_comm);
4235 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
4236 obj->ras_comm.name, r);
4240 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
4246 /* do some fini work before IP fini as dependence */
4247 int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
4249 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4251 if (!adev->ras_enabled || !con)
4255 /* Need disable ras on all IPs here before ip [hw/sw]fini */
4256 if (AMDGPU_RAS_GET_FEATURES(con->features))
4257 amdgpu_ras_disable_all_features(adev, 0);
4258 amdgpu_ras_recovery_fini(adev);
4262 int amdgpu_ras_fini(struct amdgpu_device *adev)
4264 struct amdgpu_ras_block_list *ras_node, *tmp;
4265 struct amdgpu_ras_block_object *obj = NULL;
4266 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4268 if (!adev->ras_enabled || !con)
4271 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
4272 if (ras_node->ras_obj) {
4273 obj = ras_node->ras_obj;
4274 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
4276 obj->ras_fini(adev, &obj->ras_comm);
4278 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
4281 /* Clear ras blocks from ras_list and free ras block list node */
4282 list_del(&ras_node->node);
4286 amdgpu_ras_fs_fini(adev);
4287 amdgpu_ras_interrupt_remove_all(adev);
4289 if (amdgpu_ras_aca_is_supported(adev)) {
4290 if (amdgpu_aca_is_enabled(adev))
4291 amdgpu_aca_fini(adev);
4293 amdgpu_mca_fini(adev);
4296 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared");
4298 if (AMDGPU_RAS_GET_FEATURES(con->features))
4299 amdgpu_ras_disable_all_features(adev, 0);
4301 cancel_delayed_work_sync(&con->ras_counte_delay_work);
4303 amdgpu_ras_set_context(adev, NULL);
4309 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev)
4311 struct amdgpu_ras *ras;
4313 ras = amdgpu_ras_get_context(adev);
4317 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4320 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status)
4322 struct amdgpu_ras *ras;
4324 ras = amdgpu_ras_get_context(adev);
4327 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4329 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4333 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev)
4335 struct amdgpu_ras *ras;
4337 ras = amdgpu_ras_get_context(adev);
4339 ras->ras_err_state = 0;
4342 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev,
4343 enum amdgpu_ras_block block)
4345 struct amdgpu_ras *ras;
4347 ras = amdgpu_ras_get_context(adev);
4349 set_bit(block, &ras->ras_err_state);
4352 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block)
4354 struct amdgpu_ras *ras;
4356 ras = amdgpu_ras_get_context(adev);
4358 if (block == AMDGPU_RAS_BLOCK__ANY)
4359 return (ras->ras_err_state != 0);
4361 return test_bit(block, &ras->ras_err_state) ||
4362 test_bit(AMDGPU_RAS_BLOCK__LAST,
4363 &ras->ras_err_state);
4369 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev)
4371 struct amdgpu_ras *ras;
4373 ras = amdgpu_ras_get_context(adev);
4377 return ras->event_mgr;
4380 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type,
4383 struct ras_event_manager *event_mgr;
4384 struct ras_event_state *event_state;
4387 if (type >= RAS_EVENT_TYPE_COUNT) {
4392 event_mgr = __get_ras_event_mgr(adev);
4398 event_state = &event_mgr->event_state[type];
4399 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno);
4400 atomic64_inc(&event_state->count);
4404 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n",
4405 (int)type, caller, ret);
4410 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type)
4412 struct ras_event_manager *event_mgr;
4415 if (type >= RAS_EVENT_TYPE_COUNT)
4416 return RAS_EVENT_INVALID_ID;
4419 case RAS_EVENT_TYPE_FATAL:
4420 case RAS_EVENT_TYPE_POISON_CREATION:
4421 case RAS_EVENT_TYPE_POISON_CONSUMPTION:
4422 event_mgr = __get_ras_event_mgr(adev);
4424 return RAS_EVENT_INVALID_ID;
4426 id = event_mgr->event_state[type].last_seqno;
4428 case RAS_EVENT_TYPE_INVALID:
4430 id = RAS_EVENT_INVALID_ID;
4437 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
4439 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
4440 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4441 enum ras_event_type type = RAS_EVENT_TYPE_FATAL;
4444 if (amdgpu_ras_mark_ras_event(adev, type))
4447 event_id = amdgpu_ras_acquire_event_id(adev, type);
4449 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error"
4450 "(ERREVENT_ATHUB_INTERRUPT) detected!\n");
4452 amdgpu_ras_set_fed(adev, true);
4453 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4454 amdgpu_ras_reset_gpu(adev);
4458 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
4460 if (adev->asic_type == CHIP_VEGA20 &&
4461 adev->pm.fw_version <= 0x283400) {
4462 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
4463 amdgpu_ras_intr_triggered();
4469 void amdgpu_release_ras_context(struct amdgpu_device *adev)
4471 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4476 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
4477 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
4478 amdgpu_ras_set_context(adev, NULL);
4483 #ifdef CONFIG_X86_MCE_AMD
4484 static struct amdgpu_device *find_adev(uint32_t node_id)
4487 struct amdgpu_device *adev = NULL;
4489 for (i = 0; i < mce_adev_list.num_gpu; i++) {
4490 adev = mce_adev_list.devs[i];
4492 if (adev && adev->gmc.xgmi.connected_to_cpu &&
4493 adev->gmc.xgmi.physical_node_id == node_id)
4501 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF)
4502 #define GET_UMC_INST(m) (((m) >> 21) & 0x7)
4503 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
4504 #define GPU_ID_OFFSET 8
4506 static int amdgpu_bad_page_notifier(struct notifier_block *nb,
4507 unsigned long val, void *data)
4509 struct mce *m = (struct mce *)data;
4510 struct amdgpu_device *adev = NULL;
4511 uint32_t gpu_id = 0;
4512 uint32_t umc_inst = 0, ch_inst = 0;
4515 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
4516 * and error occurred in DramECC (Extended error code = 0) then only
4517 * process the error, else bail out.
4519 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
4520 (XEC(m->status, 0x3f) == 0x0)))
4524 * If it is correctable error, return.
4526 if (mce_is_correctable(m))
4530 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
4532 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
4534 adev = find_adev(gpu_id);
4536 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
4542 * If it is uncorrectable error, then find out UMC instance and
4545 umc_inst = GET_UMC_INST(m->ipid);
4546 ch_inst = GET_CHAN_INDEX(m->ipid);
4548 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
4551 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst))
4557 static struct notifier_block amdgpu_bad_page_nb = {
4558 .notifier_call = amdgpu_bad_page_notifier,
4559 .priority = MCE_PRIO_UC,
4562 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
4565 * Add the adev to the mce_adev_list.
4566 * During mode2 reset, amdgpu device is temporarily
4567 * removed from the mgpu_info list which can cause
4568 * page retirement to fail.
4569 * Use this list instead of mgpu_info to find the amdgpu
4570 * device on which the UMC error was reported.
4572 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
4575 * Register the x86 notifier only once
4576 * with MCE subsystem.
4578 if (notifier_registered == false) {
4579 mce_register_decode_chain(&amdgpu_bad_page_nb);
4580 notifier_registered = true;
4585 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
4590 return adev->psp.ras_context.ras;
4593 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
4598 adev->psp.ras_context.ras = ras_con;
4602 /* check if ras is supported on block, say, sdma, gfx */
4603 int amdgpu_ras_is_supported(struct amdgpu_device *adev,
4607 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4609 if (block >= AMDGPU_RAS_BLOCK_COUNT)
4612 ret = ras && (adev->ras_enabled & (1 << block));
4614 /* For the special asic with mem ecc enabled but sram ecc
4615 * not enabled, even if the ras block is not supported on
4616 * .ras_enabled, if the asic supports poison mode and the
4617 * ras block has ras configuration, it can be considered
4618 * that the ras block supports ras function.
4621 (block == AMDGPU_RAS_BLOCK__GFX ||
4622 block == AMDGPU_RAS_BLOCK__SDMA ||
4623 block == AMDGPU_RAS_BLOCK__VCN ||
4624 block == AMDGPU_RAS_BLOCK__JPEG) &&
4625 (amdgpu_ras_mask & (1 << block)) &&
4626 amdgpu_ras_is_poison_mode_supported(adev) &&
4627 amdgpu_ras_get_ras_block(adev, block, 0))
4633 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
4635 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4637 /* mode1 is the only selection for RMA status */
4638 if (amdgpu_ras_is_rma(adev)) {
4639 ras->gpu_reset_flags = 0;
4640 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4643 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) {
4644 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
4645 int hive_ras_recovery = 0;
4648 hive_ras_recovery = atomic_read(&hive->ras_recovery);
4649 amdgpu_put_xgmi_hive(hive);
4651 /* In the case of multiple GPUs, after a GPU has started
4652 * resetting all GPUs on hive, other GPUs do not need to
4653 * trigger GPU reset again.
4655 if (!hive_ras_recovery)
4656 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4658 atomic_set(&ras->in_recovery, 0);
4660 flush_work(&ras->recovery_work);
4661 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4667 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable)
4669 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4673 ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4675 con->is_aca_debug_mode = enable;
4681 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable)
4683 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4687 if (amdgpu_aca_is_enabled(adev))
4688 ret = amdgpu_aca_smu_set_debug_mode(adev, enable);
4690 ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4692 con->is_aca_debug_mode = enable;
4698 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev)
4700 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4701 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
4702 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4707 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) ||
4708 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode))
4709 return con->is_aca_debug_mode;
4714 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev,
4715 unsigned int *error_query_mode)
4717 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4718 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4719 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
4722 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY;
4726 if (amdgpu_sriov_vf(adev)) {
4727 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY;
4728 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) {
4730 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY;
4732 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY;
4738 /* Register each ip ras block into amdgpu ras */
4739 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
4740 struct amdgpu_ras_block_object *ras_block_obj)
4742 struct amdgpu_ras_block_list *ras_node;
4743 if (!adev || !ras_block_obj)
4746 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
4750 INIT_LIST_HEAD(&ras_node->node);
4751 ras_node->ras_obj = ras_block_obj;
4752 list_add_tail(&ras_node->node, &adev->ras_list);
4757 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name)
4763 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE:
4764 sprintf(err_type_name, "correctable");
4766 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE:
4767 sprintf(err_type_name, "uncorrectable");
4770 sprintf(err_type_name, "unknown");
4775 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev,
4776 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4778 uint32_t *memory_id)
4780 uint32_t err_status_lo_data, err_status_lo_offset;
4785 err_status_lo_offset =
4786 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4787 reg_entry->seg_lo, reg_entry->reg_lo);
4788 err_status_lo_data = RREG32(err_status_lo_offset);
4790 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) &&
4791 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG))
4794 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID);
4799 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev,
4800 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4802 unsigned long *err_cnt)
4804 uint32_t err_status_hi_data, err_status_hi_offset;
4809 err_status_hi_offset =
4810 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4811 reg_entry->seg_hi, reg_entry->reg_hi);
4812 err_status_hi_data = RREG32(err_status_hi_offset);
4814 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) &&
4815 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG))
4816 /* keep the check here in case we need to refer to the result later */
4817 dev_dbg(adev->dev, "Invalid err_info field\n");
4819 /* read err count */
4820 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT);
4825 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev,
4826 const struct amdgpu_ras_err_status_reg_entry *reg_list,
4827 uint32_t reg_list_size,
4828 const struct amdgpu_ras_memory_id_entry *mem_list,
4829 uint32_t mem_list_size,
4832 unsigned long *err_count)
4835 unsigned long err_cnt;
4836 char err_type_name[16];
4839 for (i = 0; i < reg_list_size; i++) {
4840 /* query memory_id from err_status_lo */
4841 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i],
4842 instance, &memory_id))
4845 /* query err_cnt from err_status_hi */
4846 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i],
4847 instance, &err_cnt) ||
4851 *err_count += err_cnt;
4853 /* log the errors */
4854 amdgpu_ras_get_error_type_name(err_type, err_type_name);
4856 /* memory_list is not supported */
4858 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n",
4859 err_cnt, err_type_name,
4860 reg_list[i].block_name,
4861 instance, memory_id);
4863 for (j = 0; j < mem_list_size; j++) {
4864 if (memory_id == mem_list[j].memory_id) {
4866 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n",
4867 err_cnt, err_type_name,
4868 reg_list[i].block_name,
4869 instance, mem_list[j].name);
4877 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev,
4878 const struct amdgpu_ras_err_status_reg_entry *reg_list,
4879 uint32_t reg_list_size,
4882 uint32_t err_status_lo_offset, err_status_hi_offset;
4885 for (i = 0; i < reg_list_size; i++) {
4886 err_status_lo_offset =
4887 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4888 reg_list[i].seg_lo, reg_list[i].reg_lo);
4889 err_status_hi_offset =
4890 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4891 reg_list[i].seg_hi, reg_list[i].reg_hi);
4892 WREG32(err_status_lo_offset, 0);
4893 WREG32(err_status_hi_offset, 0);
4897 int amdgpu_ras_error_data_init(struct ras_err_data *err_data)
4899 memset(err_data, 0, sizeof(*err_data));
4901 INIT_LIST_HEAD(&err_data->err_node_list);
4906 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node)
4911 list_del(&err_node->node);
4915 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data)
4917 struct ras_err_node *err_node, *tmp;
4919 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node)
4920 amdgpu_ras_error_node_release(err_node);
4923 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data,
4924 struct amdgpu_smuio_mcm_config_info *mcm_info)
4926 struct ras_err_node *err_node;
4927 struct amdgpu_smuio_mcm_config_info *ref_id;
4929 if (!err_data || !mcm_info)
4932 for_each_ras_error(err_node, err_data) {
4933 ref_id = &err_node->err_info.mcm_info;
4935 if (mcm_info->socket_id == ref_id->socket_id &&
4936 mcm_info->die_id == ref_id->die_id)
4943 static struct ras_err_node *amdgpu_ras_error_node_new(void)
4945 struct ras_err_node *err_node;
4947 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL);
4951 INIT_LIST_HEAD(&err_node->node);
4956 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b)
4958 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node);
4959 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node);
4960 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info;
4961 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info;
4963 if (unlikely(infoa->socket_id != infob->socket_id))
4964 return infoa->socket_id - infob->socket_id;
4966 return infoa->die_id - infob->die_id;
4971 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data,
4972 struct amdgpu_smuio_mcm_config_info *mcm_info)
4974 struct ras_err_node *err_node;
4976 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info);
4978 return &err_node->err_info;
4980 err_node = amdgpu_ras_error_node_new();
4984 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info));
4986 err_data->err_list_count++;
4987 list_add_tail(&err_node->node, &err_data->err_node_list);
4988 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp);
4990 return &err_node->err_info;
4993 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data,
4994 struct amdgpu_smuio_mcm_config_info *mcm_info,
4997 struct ras_err_info *err_info;
4999 if (!err_data || !mcm_info)
5005 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5009 err_info->ue_count += count;
5010 err_data->ue_count += count;
5015 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data,
5016 struct amdgpu_smuio_mcm_config_info *mcm_info,
5019 struct ras_err_info *err_info;
5021 if (!err_data || !mcm_info)
5027 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5031 err_info->ce_count += count;
5032 err_data->ce_count += count;
5037 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data,
5038 struct amdgpu_smuio_mcm_config_info *mcm_info,
5041 struct ras_err_info *err_info;
5043 if (!err_data || !mcm_info)
5049 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5053 err_info->de_count += count;
5054 err_data->de_count += count;
5059 #define mmMP0_SMN_C2PMSG_92 0x1609C
5060 #define mmMP0_SMN_C2PMSG_126 0x160BE
5061 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev,
5064 u32 socket_id, aid_id, hbm_id;
5069 /* The pattern for smn addressing in other SOC could be different from
5070 * the one for aqua_vanjaram. We should revisit the code if the pattern
5071 * is changed. In such case, replace the aqua_vanjaram implementation
5072 * with more common helper */
5073 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5074 aqua_vanjaram_encode_ext_smn_addressing(instance);
5075 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5077 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) +
5078 aqua_vanjaram_encode_ext_smn_addressing(instance);
5079 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5081 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error);
5082 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error);
5083 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1);
5085 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error))
5087 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n",
5088 socket_id, aid_id, hbm_id, fw_status);
5090 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error))
5092 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n",
5093 socket_id, aid_id, fw_status);
5095 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error))
5097 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n",
5098 socket_id, aid_id, fw_status);
5100 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error))
5102 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n",
5103 socket_id, aid_id, fw_status);
5105 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error))
5107 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n",
5108 socket_id, aid_id, fw_status);
5110 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error))
5112 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n",
5113 socket_id, aid_id, fw_status);
5115 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error))
5117 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n",
5118 socket_id, aid_id, hbm_id, fw_status);
5120 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error))
5122 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n",
5123 socket_id, aid_id, hbm_id, fw_status);
5125 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error))
5127 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n",
5128 socket_id, aid_id, fw_status);
5130 if (AMDGPU_RAS_GPU_ERR_UNKNOWN(boot_error))
5132 "socket: %d, aid: %d, fw_status: 0x%x, unknown boot time errors\n",
5133 socket_id, aid_id, fw_status);
5136 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev,
5143 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5144 aqua_vanjaram_encode_ext_smn_addressing(instance);
5146 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) {
5147 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5148 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS)
5157 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances)
5161 for (i = 0; i < num_instances; i++) {
5162 if (amdgpu_ras_boot_error_detected(adev, i))
5163 amdgpu_ras_boot_time_error_reporting(adev, i);
5167 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn)
5169 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5170 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
5171 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT;
5174 mutex_lock(&con->page_rsv_lock);
5175 ret = amdgpu_vram_mgr_query_page_status(mgr, start);
5177 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE);
5178 mutex_unlock(&con->page_rsv_lock);
5183 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id,
5184 const char *fmt, ...)
5186 struct va_format vaf;
5189 va_start(args, fmt);
5193 if (RAS_EVENT_ID_IS_VALID(event_id))
5194 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf);
5196 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf);
5201 bool amdgpu_ras_is_rma(struct amdgpu_device *adev)
5203 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);