drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24 #include <linux/debugfs.h>
  25 #include <linux/list.h>
  26 #include <linux/module.h>
  27 #include <linux/uaccess.h>
  28 #include <linux/reboot.h>
  29 #include <linux/syscalls.h>
  30 #include <linux/pm_runtime.h>
  31 #include <linux/list_sort.h>
  32
  33 #include "amdgpu.h"
  34 #include "amdgpu_ras.h"
  35 #include "amdgpu_atomfirmware.h"
  36 #include "amdgpu_xgmi.h"
  37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
  38 #include "nbio_v4_3.h"
  39 #include "nbif_v6_3_1.h"
  40 #include "nbio_v7_9.h"
  41 #include "atom.h"
  42 #include "amdgpu_reset.h"
  43 #include "amdgpu_psp.h"
  44
  45 #ifdef CONFIG_X86_MCE_AMD
  46 #include <asm/mce.h>
  47
  48 static bool notifier_registered;
  49 #endif
  50 static const char *RAS_FS_NAME = "ras";
  51
  52 const char *ras_error_string[] = {
  53         "none",
  54         "parity",
  55         "single_correctable",
  56         "multi_uncorrectable",
  57         "poison",
  58 };
  59
  60 const char *ras_block_string[] = {
  61         "umc",
  62         "sdma",
  63         "gfx",
  64         "mmhub",
  65         "athub",
  66         "pcie_bif",
  67         "hdp",
  68         "xgmi_wafl",
  69         "df",
  70         "smn",
  71         "sem",
  72         "mp0",
  73         "mp1",
  74         "fuse",
  75         "mca",
  76         "vcn",
  77         "jpeg",
  78         "ih",
  79         "mpio",
  80 };
  81
  82 const char *ras_mca_block_string[] = {
  83         "mca_mp0",
  84         "mca_mp1",
  85         "mca_mpio",
  86         "mca_iohc",
  87 };
  88
  89 struct amdgpu_ras_block_list {
  90         /* ras block link */
  91         struct list_head node;
  92
  93         struct amdgpu_ras_block_object *ras_obj;
  94 };
  95
  96 const char *get_ras_block_str(struct ras_common_if *ras_block)
  97 {
  98         if (!ras_block)
  99                 return "NULL";
 100
 101         if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT ||
 102             ras_block->block >= ARRAY_SIZE(ras_block_string))
 103                 return "OUT OF RANGE";
 104
 105         if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
 106                 return ras_mca_block_string[ras_block->sub_block_index];
 107
 108         return ras_block_string[ras_block->block];
 109 }
 110
 111 #define ras_block_str(_BLOCK_) \
 112         (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
 113
 114 #define ras_err_str(i) (ras_error_string[ffs(i)])
 115
 116 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
 117
 118 /* inject address is 52 bits */
 119 #define RAS_UMC_INJECT_ADDR_LIMIT       (0x1ULL << 52)
 120
 121 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */
 122 #define RAS_BAD_PAGE_COVER              (100 * 1024 * 1024ULL)
 123
 124 #define MAX_UMC_POISON_POLLING_TIME_ASYNC  300  //ms
 125
 126 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100  //ms
 127
 128 #define MAX_FLUSH_RETIRE_DWORK_TIMES  100
 129
 130 enum amdgpu_ras_retire_page_reservation {
 131         AMDGPU_RAS_RETIRE_PAGE_RESERVED,
 132         AMDGPU_RAS_RETIRE_PAGE_PENDING,
 133         AMDGPU_RAS_RETIRE_PAGE_FAULT,
 134 };
 135
 136 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
 137
 138 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
 139                                 uint64_t addr);
 140 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
 141                                 uint64_t addr);
 142 #ifdef CONFIG_X86_MCE_AMD
 143 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
 144 struct mce_notifier_adev_list {
 145         struct amdgpu_device *devs[MAX_GPU_INSTANCE];
 146         int num_gpu;
 147 };
 148 static struct mce_notifier_adev_list mce_adev_list;
 149 #endif
 150
 151 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
 152 {
 153         if (adev && amdgpu_ras_get_context(adev))
 154                 amdgpu_ras_get_context(adev)->error_query_ready = ready;
 155 }
 156
 157 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
 158 {
 159         if (adev && amdgpu_ras_get_context(adev))
 160                 return amdgpu_ras_get_context(adev)->error_query_ready;
 161
 162         return false;
 163 }
 164
 165 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
 166 {
 167         struct ras_err_data err_data;
 168         struct eeprom_table_record err_rec;
 169         int ret;
 170
 171         if ((address >= adev->gmc.mc_vram_size) ||
 172             (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
 173                 dev_warn(adev->dev,
 174                          "RAS WARN: input address 0x%llx is invalid.\n",
 175                          address);
 176                 return -EINVAL;
 177         }
 178
 179         if (amdgpu_ras_check_bad_page(adev, address)) {
 180                 dev_warn(adev->dev,
 181                          "RAS WARN: 0x%llx has already been marked as bad page!\n",
 182                          address);
 183                 return 0;
 184         }
 185
 186         ret = amdgpu_ras_error_data_init(&err_data);
 187         if (ret)
 188                 return ret;
 189
 190         memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
 191         err_data.err_addr = &err_rec;
 192         amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0);
 193
 194         if (amdgpu_bad_page_threshold != 0) {
 195                 amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
 196                                          err_data.err_addr_cnt, false);
 197                 amdgpu_ras_save_bad_pages(adev, NULL);
 198         }
 199
 200         amdgpu_ras_error_data_fini(&err_data);
 201
 202         dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
 203         dev_warn(adev->dev, "Clear EEPROM:\n");
 204         dev_warn(adev->dev, "    echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
 205
 206         return 0;
 207 }
 208
 209 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
 210                                         size_t size, loff_t *pos)
 211 {
 212         struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
 213         struct ras_query_if info = {
 214                 .head = obj->head,
 215         };
 216         ssize_t s;
 217         char val[128];
 218
 219         if (amdgpu_ras_query_error_status(obj->adev, &info))
 220                 return -EINVAL;
 221
 222         /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
 223         if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
 224             amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
 225                 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
 226                         dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
 227         }
 228
 229         s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
 230                         "ue", info.ue_count,
 231                         "ce", info.ce_count);
 232         if (*pos >= s)
 233                 return 0;
 234
 235         s -= *pos;
 236         s = min_t(u64, s, size);
 237
 238
 239         if (copy_to_user(buf, &val[*pos], s))
 240                 return -EINVAL;
 241
 242         *pos += s;
 243
 244         return s;
 245 }
 246
 247 static const struct file_operations amdgpu_ras_debugfs_ops = {
 248         .owner = THIS_MODULE,
 249         .read = amdgpu_ras_debugfs_read,
 250         .write = NULL,
 251         .llseek = default_llseek
 252 };
 253
 254 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
 255 {
 256         int i;
 257
 258         for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
 259                 *block_id = i;
 260                 if (strcmp(name, ras_block_string[i]) == 0)
 261                         return 0;
 262         }
 263         return -EINVAL;
 264 }
 265
 266 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
 267                 const char __user *buf, size_t size,
 268                 loff_t *pos, struct ras_debug_if *data)
 269 {
 270         ssize_t s = min_t(u64, 64, size);
 271         char str[65];
 272         char block_name[33];
 273         char err[9] = "ue";
 274         int op = -1;
 275         int block_id;
 276         uint32_t sub_block;
 277         u64 address, value;
 278         /* default value is 0 if the mask is not set by user */
 279         u32 instance_mask = 0;
 280
 281         if (*pos)
 282                 return -EINVAL;
 283         *pos = size;
 284
 285         memset(str, 0, sizeof(str));
 286         memset(data, 0, sizeof(*data));
 287
 288         if (copy_from_user(str, buf, s))
 289                 return -EINVAL;
 290
 291         if (sscanf(str, "disable %32s", block_name) == 1)
 292                 op = 0;
 293         else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
 294                 op = 1;
 295         else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
 296                 op = 2;
 297         else if (strstr(str, "retire_page") != NULL)
 298                 op = 3;
 299         else if (str[0] && str[1] && str[2] && str[3])
 300                 /* ascii string, but commands are not matched. */
 301                 return -EINVAL;
 302
 303         if (op != -1) {
 304                 if (op == 3) {
 305                         if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
 306                             sscanf(str, "%*s %llu", &address) != 1)
 307                                 return -EINVAL;
 308
 309                         data->op = op;
 310                         data->inject.address = address;
 311
 312                         return 0;
 313                 }
 314
 315                 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
 316                         return -EINVAL;
 317
 318                 data->head.block = block_id;
 319                 /* only ue, ce and poison errors are supported */
 320                 if (!memcmp("ue", err, 2))
 321                         data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
 322                 else if (!memcmp("ce", err, 2))
 323                         data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
 324                 else if (!memcmp("poison", err, 6))
 325                         data->head.type = AMDGPU_RAS_ERROR__POISON;
 326                 else
 327                         return -EINVAL;
 328
 329                 data->op = op;
 330
 331                 if (op == 2) {
 332                         if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x",
 333                                    &sub_block, &address, &value, &instance_mask) != 4 &&
 334                             sscanf(str, "%*s %*s %*s %u %llu %llu %u",
 335                                    &sub_block, &address, &value, &instance_mask) != 4 &&
 336                                 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
 337                                    &sub_block, &address, &value) != 3 &&
 338                             sscanf(str, "%*s %*s %*s %u %llu %llu",
 339                                    &sub_block, &address, &value) != 3)
 340                                 return -EINVAL;
 341                         data->head.sub_block_index = sub_block;
 342                         data->inject.address = address;
 343                         data->inject.value = value;
 344                         data->inject.instance_mask = instance_mask;
 345                 }
 346         } else {
 347                 if (size < sizeof(*data))
 348                         return -EINVAL;
 349
 350                 if (copy_from_user(data, buf, sizeof(*data)))
 351                         return -EINVAL;
 352         }
 353
 354         return 0;
 355 }
 356
 357 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev,
 358                                 struct ras_debug_if *data)
 359 {
 360         int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
 361         uint32_t mask, inst_mask = data->inject.instance_mask;
 362
 363         /* no need to set instance mask if there is only one instance */
 364         if (num_xcc <= 1 && inst_mask) {
 365                 data->inject.instance_mask = 0;
 366                 dev_dbg(adev->dev,
 367                         "RAS inject mask(0x%x) isn't supported and force it to 0.\n",
 368                         inst_mask);
 369
 370                 return;
 371         }
 372
 373         switch (data->head.block) {
 374         case AMDGPU_RAS_BLOCK__GFX:
 375                 mask = GENMASK(num_xcc - 1, 0);
 376                 break;
 377         case AMDGPU_RAS_BLOCK__SDMA:
 378                 mask = GENMASK(adev->sdma.num_instances - 1, 0);
 379                 break;
 380         case AMDGPU_RAS_BLOCK__VCN:
 381         case AMDGPU_RAS_BLOCK__JPEG:
 382                 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0);
 383                 break;
 384         default:
 385                 mask = inst_mask;
 386                 break;
 387         }
 388
 389         /* remove invalid bits in instance mask */
 390         data->inject.instance_mask &= mask;
 391         if (inst_mask != data->inject.instance_mask)
 392                 dev_dbg(adev->dev,
 393                         "Adjust RAS inject mask 0x%x to 0x%x\n",
 394                         inst_mask, data->inject.instance_mask);
 395 }
 396
 397 /**
 398  * DOC: AMDGPU RAS debugfs control interface
 399  *
 400  * The control interface accepts struct ras_debug_if which has two members.
 401  *
 402  * First member: ras_debug_if::head or ras_debug_if::inject.
 403  *
 404  * head is used to indicate which IP block will be under control.
 405  *
 406  * head has four members, they are block, type, sub_block_index, name.
 407  * block: which IP will be under control.
 408  * type: what kind of error will be enabled/disabled/injected.
 409  * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
 410  * name: the name of IP.
 411  *
 412  * inject has three more members than head, they are address, value and mask.
 413  * As their names indicate, inject operation will write the
 414  * value to the address.
 415  *
 416  * The second member: struct ras_debug_if::op.
 417  * It has three kinds of operations.
 418  *
 419  * - 0: disable RAS on the block. Take ::head as its data.
 420  * - 1: enable RAS on the block. Take ::head as its data.
 421  * - 2: inject errors on the block. Take ::inject as its data.
 422  *
 423  * How to use the interface?
 424  *
 425  * In a program
 426  *
 427  * Copy the struct ras_debug_if in your code and initialize it.
 428  * Write the struct to the control interface.
 429  *
 430  * From shell
 431  *
 432  * .. code-block:: bash
 433  *
 434  *      echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
 435  *      echo "enable  <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
 436  *      echo "inject  <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
 437  *
 438  * Where N, is the card which you want to affect.
 439  *
 440  * "disable" requires only the block.
 441  * "enable" requires the block and error type.
 442  * "inject" requires the block, error type, address, and value.
 443  *
 444  * The block is one of: umc, sdma, gfx, etc.
 445  *      see ras_block_string[] for details
 446  *
 447  * The error type is one of: ue, ce and poison where,
 448  *      ue is multi-uncorrectable
 449  *      ce is single-correctable
 450  *      poison is poison
 451  *
 452  * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
 453  * The address and value are hexadecimal numbers, leading 0x is optional.
 454  * The mask means instance mask, is optional, default value is 0x1.
 455  *
 456  * For instance,
 457  *
 458  * .. code-block:: bash
 459  *
 460  *      echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
 461  *      echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl
 462  *      echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
 463  *
 464  * How to check the result of the operation?
 465  *
 466  * To check disable/enable, see "ras" features at,
 467  * /sys/class/drm/card[0/1/2...]/device/ras/features
 468  *
 469  * To check inject, see the corresponding error count at,
 470  * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
 471  *
 472  * .. note::
 473  *      Operations are only allowed on blocks which are supported.
 474  *      Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
 475  *      to see which blocks support RAS on a particular asic.
 476  *
 477  */
 478 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
 479                                              const char __user *buf,
 480                                              size_t size, loff_t *pos)
 481 {
 482         struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
 483         struct ras_debug_if data;
 484         int ret = 0;
 485
 486         if (!amdgpu_ras_get_error_query_ready(adev)) {
 487                 dev_warn(adev->dev, "RAS WARN: error injection "
 488                                 "currently inaccessible\n");
 489                 return size;
 490         }
 491
 492         ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
 493         if (ret)
 494                 return ret;
 495
 496         if (data.op == 3) {
 497                 ret = amdgpu_reserve_page_direct(adev, data.inject.address);
 498                 if (!ret)
 499                         return size;
 500                 else
 501                         return ret;
 502         }
 503
 504         if (!amdgpu_ras_is_supported(adev, data.head.block))
 505                 return -EINVAL;
 506
 507         switch (data.op) {
 508         case 0:
 509                 ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
 510                 break;
 511         case 1:
 512                 ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
 513                 break;
 514         case 2:
 515                 if ((data.inject.address >= adev->gmc.mc_vram_size &&
 516                     adev->gmc.mc_vram_size) ||
 517                     (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
 518                         dev_warn(adev->dev, "RAS WARN: input address "
 519                                         "0x%llx is invalid.",
 520                                         data.inject.address);
 521                         ret = -EINVAL;
 522                         break;
 523                 }
 524
 525                 /* umc ce/ue error injection for a bad page is not allowed */
 526                 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
 527                     amdgpu_ras_check_bad_page(adev, data.inject.address)) {
 528                         dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
 529                                  "already been marked as bad!\n",
 530                                  data.inject.address);
 531                         break;
 532                 }
 533
 534                 amdgpu_ras_instance_mask_check(adev, &data);
 535
 536                 /* data.inject.address is offset instead of absolute gpu address */
 537                 ret = amdgpu_ras_error_inject(adev, &data.inject);
 538                 break;
 539         default:
 540                 ret = -EINVAL;
 541                 break;
 542         }
 543
 544         if (ret)
 545                 return ret;
 546
 547         return size;
 548 }
 549
 550 /**
 551  * DOC: AMDGPU RAS debugfs EEPROM table reset interface
 552  *
 553  * Some boards contain an EEPROM which is used to persistently store a list of
 554  * bad pages which experiences ECC errors in vram.  This interface provides
 555  * a way to reset the EEPROM, e.g., after testing error injection.
 556  *
 557  * Usage:
 558  *
 559  * .. code-block:: bash
 560  *
 561  *      echo 1 > ../ras/ras_eeprom_reset
 562  *
 563  * will reset EEPROM table to 0 entries.
 564  *
 565  */
 566 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
 567                                                const char __user *buf,
 568                                                size_t size, loff_t *pos)
 569 {
 570         struct amdgpu_device *adev =
 571                 (struct amdgpu_device *)file_inode(f)->i_private;
 572         int ret;
 573
 574         ret = amdgpu_ras_eeprom_reset_table(
 575                 &(amdgpu_ras_get_context(adev)->eeprom_control));
 576
 577         if (!ret) {
 578                 /* Something was written to EEPROM.
 579                  */
 580                 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
 581                 return size;
 582         } else {
 583                 return ret;
 584         }
 585 }
 586
 587 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
 588         .owner = THIS_MODULE,
 589         .read = NULL,
 590         .write = amdgpu_ras_debugfs_ctrl_write,
 591         .llseek = default_llseek
 592 };
 593
 594 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
 595         .owner = THIS_MODULE,
 596         .read = NULL,
 597         .write = amdgpu_ras_debugfs_eeprom_write,
 598         .llseek = default_llseek
 599 };
 600
 601 /**
 602  * DOC: AMDGPU RAS sysfs Error Count Interface
 603  *
 604  * It allows the user to read the error count for each IP block on the gpu through
 605  * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
 606  *
 607  * It outputs the multiple lines which report the uncorrected (ue) and corrected
 608  * (ce) error counts.
 609  *
 610  * The format of one line is below,
 611  *
 612  * [ce|ue]: count
 613  *
 614  * Example:
 615  *
 616  * .. code-block:: bash
 617  *
 618  *      ue: 0
 619  *      ce: 1
 620  *
 621  */
 622 static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
 623                 struct device_attribute *attr, char *buf)
 624 {
 625         struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
 626         struct ras_query_if info = {
 627                 .head = obj->head,
 628         };
 629
 630         if (!amdgpu_ras_get_error_query_ready(obj->adev))
 631                 return sysfs_emit(buf, "Query currently inaccessible\n");
 632
 633         if (amdgpu_ras_query_error_status(obj->adev, &info))
 634                 return -EINVAL;
 635
 636         if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
 637             amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
 638                 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
 639                         dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
 640         }
 641
 642         if (info.head.block == AMDGPU_RAS_BLOCK__UMC)
 643                 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
 644                                 "ce", info.ce_count, "de", info.de_count);
 645         else
 646                 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
 647                                 "ce", info.ce_count);
 648 }
 649
 650 /* obj begin */
 651
 652 #define get_obj(obj) do { (obj)->use++; } while (0)
 653 #define alive_obj(obj) ((obj)->use)
 654
 655 static inline void put_obj(struct ras_manager *obj)
 656 {
 657         if (obj && (--obj->use == 0)) {
 658                 list_del(&obj->node);
 659                 amdgpu_ras_error_data_fini(&obj->err_data);
 660         }
 661
 662         if (obj && (obj->use < 0))
 663                 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
 664 }
 665
 666 /* make one obj and return it. */
 667 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
 668                 struct ras_common_if *head)
 669 {
 670         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 671         struct ras_manager *obj;
 672
 673         if (!adev->ras_enabled || !con)
 674                 return NULL;
 675
 676         if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
 677                 return NULL;
 678
 679         if (head->block == AMDGPU_RAS_BLOCK__MCA) {
 680                 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
 681                         return NULL;
 682
 683                 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
 684         } else
 685                 obj = &con->objs[head->block];
 686
 687         /* already exist. return obj? */
 688         if (alive_obj(obj))
 689                 return NULL;
 690
 691         if (amdgpu_ras_error_data_init(&obj->err_data))
 692                 return NULL;
 693
 694         obj->head = *head;
 695         obj->adev = adev;
 696         list_add(&obj->node, &con->head);
 697         get_obj(obj);
 698
 699         return obj;
 700 }
 701
 702 /* return an obj equal to head, or the first when head is NULL */
 703 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
 704                 struct ras_common_if *head)
 705 {
 706         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 707         struct ras_manager *obj;
 708         int i;
 709
 710         if (!adev->ras_enabled || !con)
 711                 return NULL;
 712
 713         if (head) {
 714                 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
 715                         return NULL;
 716
 717                 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
 718                         if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
 719                                 return NULL;
 720
 721                         obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
 722                 } else
 723                         obj = &con->objs[head->block];
 724
 725                 if (alive_obj(obj))
 726                         return obj;
 727         } else {
 728                 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
 729                         obj = &con->objs[i];
 730                         if (alive_obj(obj))
 731                                 return obj;
 732                 }
 733         }
 734
 735         return NULL;
 736 }
 737 /* obj end */
 738
 739 /* feature ctl begin */
 740 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
 741                                          struct ras_common_if *head)
 742 {
 743         return adev->ras_hw_enabled & BIT(head->block);
 744 }
 745
 746 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
 747                 struct ras_common_if *head)
 748 {
 749         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 750
 751         return con->features & BIT(head->block);
 752 }
 753
 754 /*
 755  * if obj is not created, then create one.
 756  * set feature enable flag.
 757  */
 758 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
 759                 struct ras_common_if *head, int enable)
 760 {
 761         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 762         struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
 763
 764         /* If hardware does not support ras, then do not create obj.
 765          * But if hardware support ras, we can create the obj.
 766          * Ras framework checks con->hw_supported to see if it need do
 767          * corresponding initialization.
 768          * IP checks con->support to see if it need disable ras.
 769          */
 770         if (!amdgpu_ras_is_feature_allowed(adev, head))
 771                 return 0;
 772
 773         if (enable) {
 774                 if (!obj) {
 775                         obj = amdgpu_ras_create_obj(adev, head);
 776                         if (!obj)
 777                                 return -EINVAL;
 778                 } else {
 779                         /* In case we create obj somewhere else */
 780                         get_obj(obj);
 781                 }
 782                 con->features |= BIT(head->block);
 783         } else {
 784                 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
 785                         con->features &= ~BIT(head->block);
 786                         put_obj(obj);
 787                 }
 788         }
 789
 790         return 0;
 791 }
 792
 793 /* wrapper of psp_ras_enable_features */
 794 int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
 795                 struct ras_common_if *head, bool enable)
 796 {
 797         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 798         union ta_ras_cmd_input *info;
 799         int ret;
 800
 801         if (!con)
 802                 return -EINVAL;
 803
 804         /* For non-gfx ip, do not enable ras feature if it is not allowed */
 805         /* For gfx ip, regardless of feature support status, */
 806         /* Force issue enable or disable ras feature commands */
 807         if (head->block != AMDGPU_RAS_BLOCK__GFX &&
 808             !amdgpu_ras_is_feature_allowed(adev, head))
 809                 return 0;
 810
 811         /* Only enable gfx ras feature from host side */
 812         if (head->block == AMDGPU_RAS_BLOCK__GFX &&
 813             !amdgpu_sriov_vf(adev) &&
 814             !amdgpu_ras_intr_triggered()) {
 815                 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
 816                 if (!info)
 817                         return -ENOMEM;
 818
 819                 if (!enable) {
 820                         info->disable_features = (struct ta_ras_disable_features_input) {
 821                                 .block_id =  amdgpu_ras_block_to_ta(head->block),
 822                                 .error_type = amdgpu_ras_error_to_ta(head->type),
 823                         };
 824                 } else {
 825                         info->enable_features = (struct ta_ras_enable_features_input) {
 826                                 .block_id =  amdgpu_ras_block_to_ta(head->block),
 827                                 .error_type = amdgpu_ras_error_to_ta(head->type),
 828                         };
 829                 }
 830
 831                 ret = psp_ras_enable_features(&adev->psp, info, enable);
 832                 if (ret) {
 833                         dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
 834                                 enable ? "enable":"disable",
 835                                 get_ras_block_str(head),
 836                                 amdgpu_ras_is_poison_mode_supported(adev), ret);
 837                         kfree(info);
 838                         return ret;
 839                 }
 840
 841                 kfree(info);
 842         }
 843
 844         /* setup the obj */
 845         __amdgpu_ras_feature_enable(adev, head, enable);
 846
 847         return 0;
 848 }
 849
 850 /* Only used in device probe stage and called only once. */
 851 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
 852                 struct ras_common_if *head, bool enable)
 853 {
 854         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 855         int ret;
 856
 857         if (!con)
 858                 return -EINVAL;
 859
 860         if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
 861                 if (enable) {
 862                         /* There is no harm to issue a ras TA cmd regardless of
 863                          * the currecnt ras state.
 864                          * If current state == target state, it will do nothing
 865                          * But sometimes it requests driver to reset and repost
 866                          * with error code -EAGAIN.
 867                          */
 868                         ret = amdgpu_ras_feature_enable(adev, head, 1);
 869                         /* With old ras TA, we might fail to enable ras.
 870                          * Log it and just setup the object.
 871                          * TODO need remove this WA in the future.
 872                          */
 873                         if (ret == -EINVAL) {
 874                                 ret = __amdgpu_ras_feature_enable(adev, head, 1);
 875                                 if (!ret)
 876                                         dev_info(adev->dev,
 877                                                 "RAS INFO: %s setup object\n",
 878                                                 get_ras_block_str(head));
 879                         }
 880                 } else {
 881                         /* setup the object then issue a ras TA disable cmd.*/
 882                         ret = __amdgpu_ras_feature_enable(adev, head, 1);
 883                         if (ret)
 884                                 return ret;
 885
 886                         /* gfx block ras disable cmd must send to ras-ta */
 887                         if (head->block == AMDGPU_RAS_BLOCK__GFX)
 888                                 con->features |= BIT(head->block);
 889
 890                         ret = amdgpu_ras_feature_enable(adev, head, 0);
 891
 892                         /* clean gfx block ras features flag */
 893                         if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
 894                                 con->features &= ~BIT(head->block);
 895                 }
 896         } else
 897                 ret = amdgpu_ras_feature_enable(adev, head, enable);
 898
 899         return ret;
 900 }
 901
 902 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
 903                 bool bypass)
 904 {
 905         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 906         struct ras_manager *obj, *tmp;
 907
 908         list_for_each_entry_safe(obj, tmp, &con->head, node) {
 909                 /* bypass psp.
 910                  * aka just release the obj and corresponding flags
 911                  */
 912                 if (bypass) {
 913                         if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
 914                                 break;
 915                 } else {
 916                         if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
 917                                 break;
 918                 }
 919         }
 920
 921         return con->features;
 922 }
 923
 924 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
 925                 bool bypass)
 926 {
 927         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 928         int i;
 929         const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
 930
 931         for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
 932                 struct ras_common_if head = {
 933                         .block = i,
 934                         .type = default_ras_type,
 935                         .sub_block_index = 0,
 936                 };
 937
 938                 if (i == AMDGPU_RAS_BLOCK__MCA)
 939                         continue;
 940
 941                 if (bypass) {
 942                         /*
 943                          * bypass psp. vbios enable ras for us.
 944                          * so just create the obj
 945                          */
 946                         if (__amdgpu_ras_feature_enable(adev, &head, 1))
 947                                 break;
 948                 } else {
 949                         if (amdgpu_ras_feature_enable(adev, &head, 1))
 950                                 break;
 951                 }
 952         }
 953
 954         for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
 955                 struct ras_common_if head = {
 956                         .block = AMDGPU_RAS_BLOCK__MCA,
 957                         .type = default_ras_type,
 958                         .sub_block_index = i,
 959                 };
 960
 961                 if (bypass) {
 962                         /*
 963                          * bypass psp. vbios enable ras for us.
 964                          * so just create the obj
 965                          */
 966                         if (__amdgpu_ras_feature_enable(adev, &head, 1))
 967                                 break;
 968                 } else {
 969                         if (amdgpu_ras_feature_enable(adev, &head, 1))
 970                                 break;
 971                 }
 972         }
 973
 974         return con->features;
 975 }
 976 /* feature ctl end */
 977
 978 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
 979                 enum amdgpu_ras_block block)
 980 {
 981         if (!block_obj)
 982                 return -EINVAL;
 983
 984         if (block_obj->ras_comm.block == block)
 985                 return 0;
 986
 987         return -EINVAL;
 988 }
 989
 990 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
 991                                         enum amdgpu_ras_block block, uint32_t sub_block_index)
 992 {
 993         struct amdgpu_ras_block_list *node, *tmp;
 994         struct amdgpu_ras_block_object *obj;
 995
 996         if (block >= AMDGPU_RAS_BLOCK__LAST)
 997                 return NULL;
 998
 999         list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
1000                 if (!node->ras_obj) {
1001                         dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
1002                         continue;
1003                 }
1004
1005                 obj = node->ras_obj;
1006                 if (obj->ras_block_match) {
1007                         if (obj->ras_block_match(obj, block, sub_block_index) == 0)
1008                                 return obj;
1009                 } else {
1010                         if (amdgpu_ras_block_match_default(obj, block) == 0)
1011                                 return obj;
1012                 }
1013         }
1014
1015         return NULL;
1016 }
1017
1018 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
1019 {
1020         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
1021         int ret = 0;
1022
1023         /*
1024          * choosing right query method according to
1025          * whether smu support query error information
1026          */
1027         ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
1028         if (ret == -EOPNOTSUPP) {
1029                 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1030                         adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
1031                         adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
1032
1033                 /* umc query_ras_error_address is also responsible for clearing
1034                  * error status
1035                  */
1036                 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1037                     adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
1038                         adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
1039         } else if (!ret) {
1040                 if (adev->umc.ras &&
1041                         adev->umc.ras->ecc_info_query_ras_error_count)
1042                         adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
1043
1044                 if (adev->umc.ras &&
1045                         adev->umc.ras->ecc_info_query_ras_error_address)
1046                         adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
1047         }
1048 }
1049
1050 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
1051                                               struct ras_manager *ras_mgr,
1052                                               struct ras_err_data *err_data,
1053                                               struct ras_query_context *qctx,
1054                                               const char *blk_name,
1055                                               bool is_ue,
1056                                               bool is_de)
1057 {
1058         struct amdgpu_smuio_mcm_config_info *mcm_info;
1059         struct ras_err_node *err_node;
1060         struct ras_err_info *err_info;
1061         u64 event_id = qctx->evid.event_id;
1062
1063         if (is_ue) {
1064                 for_each_ras_error(err_node, err_data) {
1065                         err_info = &err_node->err_info;
1066                         mcm_info = &err_info->mcm_info;
1067                         if (err_info->ue_count) {
1068                                 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1069                                               "%lld new uncorrectable hardware errors detected in %s block\n",
1070                                               mcm_info->socket_id,
1071                                               mcm_info->die_id,
1072                                               err_info->ue_count,
1073                                               blk_name);
1074                         }
1075                 }
1076
1077                 for_each_ras_error(err_node, &ras_mgr->err_data) {
1078                         err_info = &err_node->err_info;
1079                         mcm_info = &err_info->mcm_info;
1080                         RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1081                                       "%lld uncorrectable hardware errors detected in total in %s block\n",
1082                                       mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name);
1083                 }
1084
1085         } else {
1086                 if (is_de) {
1087                         for_each_ras_error(err_node, err_data) {
1088                                 err_info = &err_node->err_info;
1089                                 mcm_info = &err_info->mcm_info;
1090                                 if (err_info->de_count) {
1091                                         RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1092                                                       "%lld new deferred hardware errors detected in %s block\n",
1093                                                       mcm_info->socket_id,
1094                                                       mcm_info->die_id,
1095                                                       err_info->de_count,
1096                                                       blk_name);
1097                                 }
1098                         }
1099
1100                         for_each_ras_error(err_node, &ras_mgr->err_data) {
1101                                 err_info = &err_node->err_info;
1102                                 mcm_info = &err_info->mcm_info;
1103                                 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1104                                               "%lld deferred hardware errors detected in total in %s block\n",
1105                                               mcm_info->socket_id, mcm_info->die_id,
1106                                               err_info->de_count, blk_name);
1107                         }
1108                 } else {
1109                         for_each_ras_error(err_node, err_data) {
1110                                 err_info = &err_node->err_info;
1111                                 mcm_info = &err_info->mcm_info;
1112                                 if (err_info->ce_count) {
1113                                         RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1114                                                       "%lld new correctable hardware errors detected in %s block\n",
1115                                                       mcm_info->socket_id,
1116                                                       mcm_info->die_id,
1117                                                       err_info->ce_count,
1118                                                       blk_name);
1119                                 }
1120                         }
1121
1122                         for_each_ras_error(err_node, &ras_mgr->err_data) {
1123                                 err_info = &err_node->err_info;
1124                                 mcm_info = &err_info->mcm_info;
1125                                 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1126                                               "%lld correctable hardware errors detected in total in %s block\n",
1127                                               mcm_info->socket_id, mcm_info->die_id,
1128                                               err_info->ce_count, blk_name);
1129                         }
1130                 }
1131         }
1132 }
1133
1134 static inline bool err_data_has_source_info(struct ras_err_data *data)
1135 {
1136         return !list_empty(&data->err_node_list);
1137 }
1138
1139 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
1140                                              struct ras_query_if *query_if,
1141                                              struct ras_err_data *err_data,
1142                                              struct ras_query_context *qctx)
1143 {
1144         struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head);
1145         const char *blk_name = get_ras_block_str(&query_if->head);
1146         u64 event_id = qctx->evid.event_id;
1147
1148         if (err_data->ce_count) {
1149                 if (err_data_has_source_info(err_data)) {
1150                         amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1151                                                           blk_name, false, false);
1152                 } else if (!adev->aid_mask &&
1153                            adev->smuio.funcs &&
1154                            adev->smuio.funcs->get_socket_id &&
1155                            adev->smuio.funcs->get_die_id) {
1156                         RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1157                                       "%ld correctable hardware errors "
1158                                       "detected in %s block\n",
1159                                       adev->smuio.funcs->get_socket_id(adev),
1160                                       adev->smuio.funcs->get_die_id(adev),
1161                                       ras_mgr->err_data.ce_count,
1162                                       blk_name);
1163                 } else {
1164                         RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors "
1165                                       "detected in %s block\n",
1166                                       ras_mgr->err_data.ce_count,
1167                                       blk_name);
1168                 }
1169         }
1170
1171         if (err_data->ue_count) {
1172                 if (err_data_has_source_info(err_data)) {
1173                         amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1174                                                           blk_name, true, false);
1175                 } else if (!adev->aid_mask &&
1176                            adev->smuio.funcs &&
1177                            adev->smuio.funcs->get_socket_id &&
1178                            adev->smuio.funcs->get_die_id) {
1179                         RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1180                                       "%ld uncorrectable hardware errors "
1181                                       "detected in %s block\n",
1182                                       adev->smuio.funcs->get_socket_id(adev),
1183                                       adev->smuio.funcs->get_die_id(adev),
1184                                       ras_mgr->err_data.ue_count,
1185                                       blk_name);
1186                 } else {
1187                         RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors "
1188                                       "detected in %s block\n",
1189                                       ras_mgr->err_data.ue_count,
1190                                       blk_name);
1191                 }
1192         }
1193
1194         if (err_data->de_count) {
1195                 if (err_data_has_source_info(err_data)) {
1196                         amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1197                                                           blk_name, false, true);
1198                 } else if (!adev->aid_mask &&
1199                            adev->smuio.funcs &&
1200                            adev->smuio.funcs->get_socket_id &&
1201                            adev->smuio.funcs->get_die_id) {
1202                         RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1203                                       "%ld deferred hardware errors "
1204                                       "detected in %s block\n",
1205                                       adev->smuio.funcs->get_socket_id(adev),
1206                                       adev->smuio.funcs->get_die_id(adev),
1207                                       ras_mgr->err_data.de_count,
1208                                       blk_name);
1209                 } else {
1210                         RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors "
1211                                       "detected in %s block\n",
1212                                       ras_mgr->err_data.de_count,
1213                                       blk_name);
1214                 }
1215         }
1216 }
1217
1218 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev,
1219                                                   struct ras_query_if *query_if,
1220                                                   struct ras_err_data *err_data,
1221                                                   struct ras_query_context *qctx)
1222 {
1223         unsigned long new_ue, new_ce, new_de;
1224         struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head);
1225         const char *blk_name = get_ras_block_str(&query_if->head);
1226         u64 event_id = qctx->evid.event_id;
1227
1228         new_ce = err_data->ce_count - obj->err_data.ce_count;
1229         new_ue = err_data->ue_count - obj->err_data.ue_count;
1230         new_de = err_data->de_count - obj->err_data.de_count;
1231
1232         if (new_ce) {
1233                 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors "
1234                               "detected in %s block\n",
1235                               new_ce,
1236                               blk_name);
1237         }
1238
1239         if (new_ue) {
1240                 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors "
1241                               "detected in %s block\n",
1242                               new_ue,
1243                               blk_name);
1244         }
1245
1246         if (new_de) {
1247                 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors "
1248                               "detected in %s block\n",
1249                               new_de,
1250                               blk_name);
1251         }
1252 }
1253
1254 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data)
1255 {
1256         struct ras_err_node *err_node;
1257         struct ras_err_info *err_info;
1258
1259         if (err_data_has_source_info(err_data)) {
1260                 for_each_ras_error(err_node, err_data) {
1261                         err_info = &err_node->err_info;
1262                         amdgpu_ras_error_statistic_de_count(&obj->err_data,
1263                                         &err_info->mcm_info, err_info->de_count);
1264                         amdgpu_ras_error_statistic_ce_count(&obj->err_data,
1265                                         &err_info->mcm_info, err_info->ce_count);
1266                         amdgpu_ras_error_statistic_ue_count(&obj->err_data,
1267                                         &err_info->mcm_info, err_info->ue_count);
1268                 }
1269         } else {
1270                 /* for legacy asic path which doesn't has error source info */
1271                 obj->err_data.ue_count += err_data->ue_count;
1272                 obj->err_data.ce_count += err_data->ce_count;
1273                 obj->err_data.de_count += err_data->de_count;
1274         }
1275 }
1276
1277 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj,
1278                                                              struct ras_err_data *err_data)
1279 {
1280         /* Host reports absolute counts */
1281         obj->err_data.ue_count = err_data->ue_count;
1282         obj->err_data.ce_count = err_data->ce_count;
1283         obj->err_data.de_count = err_data->de_count;
1284 }
1285
1286 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1287 {
1288         struct ras_common_if head;
1289
1290         memset(&head, 0, sizeof(head));
1291         head.block = blk;
1292
1293         return amdgpu_ras_find_obj(adev, &head);
1294 }
1295
1296 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1297                         const struct aca_info *aca_info, void *data)
1298 {
1299         struct ras_manager *obj;
1300
1301         /* in resume phase, no need to create aca fs node */
1302         if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
1303                 return 0;
1304
1305         obj = get_ras_manager(adev, blk);
1306         if (!obj)
1307                 return -EINVAL;
1308
1309         return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data);
1310 }
1311
1312 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1313 {
1314         struct ras_manager *obj;
1315
1316         obj = get_ras_manager(adev, blk);
1317         if (!obj)
1318                 return -EINVAL;
1319
1320         amdgpu_aca_remove_handle(&obj->aca_handle);
1321
1322         return 0;
1323 }
1324
1325 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1326                                          enum aca_error_type type, struct ras_err_data *err_data,
1327                                          struct ras_query_context *qctx)
1328 {
1329         struct ras_manager *obj;
1330
1331         obj = get_ras_manager(adev, blk);
1332         if (!obj)
1333                 return -EINVAL;
1334
1335         return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx);
1336 }
1337
1338 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr,
1339                                   struct aca_handle *handle, char *buf, void *data)
1340 {
1341         struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle);
1342         struct ras_query_if info = {
1343                 .head = obj->head,
1344         };
1345
1346         if (!amdgpu_ras_get_error_query_ready(obj->adev))
1347                 return sysfs_emit(buf, "Query currently inaccessible\n");
1348
1349         if (amdgpu_ras_query_error_status(obj->adev, &info))
1350                 return -EINVAL;
1351
1352         return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
1353                           "ce", info.ce_count, "de", info.de_count);
1354 }
1355
1356 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
1357                                                 struct ras_query_if *info,
1358                                                 struct ras_err_data *err_data,
1359                                                 struct ras_query_context *qctx,
1360                                                 unsigned int error_query_mode)
1361 {
1362         enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT;
1363         struct amdgpu_ras_block_object *block_obj = NULL;
1364         int ret;
1365
1366         if (blk == AMDGPU_RAS_BLOCK_COUNT)
1367                 return -EINVAL;
1368
1369         if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY)
1370                 return -EINVAL;
1371
1372         if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1373                 return amdgpu_virt_req_ras_err_count(adev, blk, err_data);
1374         } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
1375                 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
1376                         amdgpu_ras_get_ecc_info(adev, err_data);
1377                 } else {
1378                         block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
1379                         if (!block_obj || !block_obj->hw_ops) {
1380                                 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1381                                              get_ras_block_str(&info->head));
1382                                 return -EINVAL;
1383                         }
1384
1385                         if (block_obj->hw_ops->query_ras_error_count)
1386                                 block_obj->hw_ops->query_ras_error_count(adev, err_data);
1387
1388                         if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
1389                             (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
1390                             (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
1391                                 if (block_obj->hw_ops->query_ras_error_status)
1392                                         block_obj->hw_ops->query_ras_error_status(adev);
1393                         }
1394                 }
1395         } else {
1396                 if (amdgpu_aca_is_enabled(adev)) {
1397                         ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx);
1398                         if (ret)
1399                                 return ret;
1400
1401                         ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx);
1402                         if (ret)
1403                                 return ret;
1404
1405                         ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx);
1406                         if (ret)
1407                                 return ret;
1408                 } else {
1409                         /* FIXME: add code to check return value later */
1410                         amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx);
1411                         amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx);
1412                 }
1413         }
1414
1415         return 0;
1416 }
1417
1418 /* query/inject/cure begin */
1419 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev,
1420                                                     struct ras_query_if *info,
1421                                                     enum ras_event_type type)
1422 {
1423         struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1424         struct ras_err_data err_data;
1425         struct ras_query_context qctx;
1426         unsigned int error_query_mode;
1427         int ret;
1428
1429         if (!obj)
1430                 return -EINVAL;
1431
1432         ret = amdgpu_ras_error_data_init(&err_data);
1433         if (ret)
1434                 return ret;
1435
1436         if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode))
1437                 return -EINVAL;
1438
1439         memset(&qctx, 0, sizeof(qctx));
1440         qctx.evid.type = type;
1441         qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type);
1442
1443         if (!down_read_trylock(&adev->reset_domain->sem)) {
1444                 ret = -EIO;
1445                 goto out_fini_err_data;
1446         }
1447
1448         ret = amdgpu_ras_query_error_status_helper(adev, info,
1449                                                    &err_data,
1450                                                    &qctx,
1451                                                    error_query_mode);
1452         up_read(&adev->reset_domain->sem);
1453         if (ret)
1454                 goto out_fini_err_data;
1455
1456         if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1457                 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data);
1458                 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx);
1459         } else {
1460                 /* Host provides absolute error counts. First generate the report
1461                  * using the previous VF internal count against new host count.
1462                  * Then Update VF internal count.
1463                  */
1464                 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx);
1465                 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data);
1466         }
1467
1468         info->ue_count = obj->err_data.ue_count;
1469         info->ce_count = obj->err_data.ce_count;
1470         info->de_count = obj->err_data.de_count;
1471
1472 out_fini_err_data:
1473         amdgpu_ras_error_data_fini(&err_data);
1474
1475         return ret;
1476 }
1477
1478 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info)
1479 {
1480         return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID);
1481 }
1482
1483 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev,
1484                 enum amdgpu_ras_block block)
1485 {
1486         struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1487         const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
1488         const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
1489
1490         if (!block_obj || !block_obj->hw_ops) {
1491                 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1492                                 ras_block_str(block));
1493                 return -EOPNOTSUPP;
1494         }
1495
1496         if (!amdgpu_ras_is_supported(adev, block) ||
1497             !amdgpu_ras_get_aca_debug_mode(adev))
1498                 return -EOPNOTSUPP;
1499
1500         /* skip ras error reset in gpu reset */
1501         if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) &&
1502             ((smu_funcs && smu_funcs->set_debug_mode) ||
1503              (mca_funcs && mca_funcs->mca_set_debug_mode)))
1504                 return -EOPNOTSUPP;
1505
1506         if (block_obj->hw_ops->reset_ras_error_count)
1507                 block_obj->hw_ops->reset_ras_error_count(adev);
1508
1509         return 0;
1510 }
1511
1512 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1513                 enum amdgpu_ras_block block)
1514 {
1515         struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1516
1517         if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP)
1518                 return 0;
1519
1520         if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1521             (block == AMDGPU_RAS_BLOCK__MMHUB)) {
1522                 if (block_obj->hw_ops->reset_ras_error_status)
1523                         block_obj->hw_ops->reset_ras_error_status(adev);
1524         }
1525
1526         return 0;
1527 }
1528
1529 /* wrapper of psp_ras_trigger_error */
1530 int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1531                 struct ras_inject_if *info)
1532 {
1533         struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1534         struct ta_ras_trigger_error_input block_info = {
1535                 .block_id =  amdgpu_ras_block_to_ta(info->head.block),
1536                 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
1537                 .sub_block_index = info->head.sub_block_index,
1538                 .address = info->address,
1539                 .value = info->value,
1540         };
1541         int ret = -EINVAL;
1542         struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1543                                                         info->head.block,
1544                                                         info->head.sub_block_index);
1545
1546         /* inject on guest isn't allowed, return success directly */
1547         if (amdgpu_sriov_vf(adev))
1548                 return 0;
1549
1550         if (!obj)
1551                 return -EINVAL;
1552
1553         if (!block_obj || !block_obj->hw_ops)   {
1554                 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1555                              get_ras_block_str(&info->head));
1556                 return -EINVAL;
1557         }
1558
1559         /* Calculate XGMI relative offset */
1560         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1561             info->head.block != AMDGPU_RAS_BLOCK__GFX) {
1562                 block_info.address =
1563                         amdgpu_xgmi_get_relative_phy_addr(adev,
1564                                                           block_info.address);
1565         }
1566
1567         if (block_obj->hw_ops->ras_error_inject) {
1568                 if (info->head.block == AMDGPU_RAS_BLOCK__GFX)
1569                         ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask);
1570                 else /* Special ras_error_inject is defined (e.g: xgmi) */
1571                         ret = block_obj->hw_ops->ras_error_inject(adev, &block_info,
1572                                                 info->instance_mask);
1573         } else {
1574                 /* default path */
1575                 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask);
1576         }
1577
1578         if (ret)
1579                 dev_err(adev->dev, "ras inject %s failed %d\n",
1580                         get_ras_block_str(&info->head), ret);
1581
1582         return ret;
1583 }
1584
1585 /**
1586  * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP
1587  * @adev: pointer to AMD GPU device
1588  * @ce_count: pointer to an integer to be set to the count of correctible errors.
1589  * @ue_count: pointer to an integer to be set to the count of uncorrectible errors.
1590  * @query_info: pointer to ras_query_if
1591  *
1592  * Return 0 for query success or do nothing, otherwise return an error
1593  * on failures
1594  */
1595 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev,
1596                                                unsigned long *ce_count,
1597                                                unsigned long *ue_count,
1598                                                struct ras_query_if *query_info)
1599 {
1600         int ret;
1601
1602         if (!query_info)
1603                 /* do nothing if query_info is not specified */
1604                 return 0;
1605
1606         ret = amdgpu_ras_query_error_status(adev, query_info);
1607         if (ret)
1608                 return ret;
1609
1610         *ce_count += query_info->ce_count;
1611         *ue_count += query_info->ue_count;
1612
1613         /* some hardware/IP supports read to clear
1614          * no need to explictly reset the err status after the query call */
1615         if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
1616             amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
1617                 if (amdgpu_ras_reset_error_status(adev, query_info->head.block))
1618                         dev_warn(adev->dev,
1619                                  "Failed to reset error counter and error status\n");
1620         }
1621
1622         return 0;
1623 }
1624
1625 /**
1626  * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP
1627  * @adev: pointer to AMD GPU device
1628  * @ce_count: pointer to an integer to be set to the count of correctible errors.
1629  * @ue_count: pointer to an integer to be set to the count of uncorrectible
1630  * errors.
1631  * @query_info: pointer to ras_query_if if the query request is only for
1632  * specific ip block; if info is NULL, then the qurey request is for
1633  * all the ip blocks that support query ras error counters/status
1634  *
1635  * If set, @ce_count or @ue_count, count and return the corresponding
1636  * error counts in those integer pointers. Return 0 if the device
1637  * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1638  */
1639 int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1640                                  unsigned long *ce_count,
1641                                  unsigned long *ue_count,
1642                                  struct ras_query_if *query_info)
1643 {
1644         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1645         struct ras_manager *obj;
1646         unsigned long ce, ue;
1647         int ret;
1648
1649         if (!adev->ras_enabled || !con)
1650                 return -EOPNOTSUPP;
1651
1652         /* Don't count since no reporting.
1653          */
1654         if (!ce_count && !ue_count)
1655                 return 0;
1656
1657         ce = 0;
1658         ue = 0;
1659         if (!query_info) {
1660                 /* query all the ip blocks that support ras query interface */
1661                 list_for_each_entry(obj, &con->head, node) {
1662                         struct ras_query_if info = {
1663                                 .head = obj->head,
1664                         };
1665
1666                         ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info);
1667                 }
1668         } else {
1669                 /* query specific ip block */
1670                 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info);
1671         }
1672
1673         if (ret)
1674                 return ret;
1675
1676         if (ce_count)
1677                 *ce_count = ce;
1678
1679         if (ue_count)
1680                 *ue_count = ue;
1681
1682         return 0;
1683 }
1684 /* query/inject/cure end */
1685
1686
1687 /* sysfs begin */
1688
1689 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1690                 struct ras_badpage **bps, unsigned int *count);
1691
1692 static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1693 {
1694         switch (flags) {
1695         case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
1696                 return "R";
1697         case AMDGPU_RAS_RETIRE_PAGE_PENDING:
1698                 return "P";
1699         case AMDGPU_RAS_RETIRE_PAGE_FAULT:
1700         default:
1701                 return "F";
1702         }
1703 }
1704
1705 /**
1706  * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
1707  *
1708  * It allows user to read the bad pages of vram on the gpu through
1709  * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1710  *
1711  * It outputs multiple lines, and each line stands for one gpu page.
1712  *
1713  * The format of one line is below,
1714  * gpu pfn : gpu page size : flags
1715  *
1716  * gpu pfn and gpu page size are printed in hex format.
1717  * flags can be one of below character,
1718  *
1719  * R: reserved, this gpu page is reserved and not able to use.
1720  *
1721  * P: pending for reserve, this gpu page is marked as bad, will be reserved
1722  * in next window of page_reserve.
1723  *
1724  * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1725  *
1726  * Examples:
1727  *
1728  * .. code-block:: bash
1729  *
1730  *      0x00000001 : 0x00001000 : R
1731  *      0x00000002 : 0x00001000 : P
1732  *
1733  */
1734
1735 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1736                 struct kobject *kobj, struct bin_attribute *attr,
1737                 char *buf, loff_t ppos, size_t count)
1738 {
1739         struct amdgpu_ras *con =
1740                 container_of(attr, struct amdgpu_ras, badpages_attr);
1741         struct amdgpu_device *adev = con->adev;
1742         const unsigned int element_size =
1743                 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
1744         unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1745         unsigned int end = div64_ul(ppos + count - 1, element_size);
1746         ssize_t s = 0;
1747         struct ras_badpage *bps = NULL;
1748         unsigned int bps_count = 0;
1749
1750         memset(buf, 0, count);
1751
1752         if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1753                 return 0;
1754
1755         for (; start < end && start < bps_count; start++)
1756                 s += scnprintf(&buf[s], element_size + 1,
1757                                 "0x%08x : 0x%08x : %1s\n",
1758                                 bps[start].bp,
1759                                 bps[start].size,
1760                                 amdgpu_ras_badpage_flags_str(bps[start].flags));
1761
1762         kfree(bps);
1763
1764         return s;
1765 }
1766
1767 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1768                 struct device_attribute *attr, char *buf)
1769 {
1770         struct amdgpu_ras *con =
1771                 container_of(attr, struct amdgpu_ras, features_attr);
1772
1773         return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
1774 }
1775
1776 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev,
1777                 struct device_attribute *attr, char *buf)
1778 {
1779         struct amdgpu_ras *con =
1780                 container_of(attr, struct amdgpu_ras, version_attr);
1781         return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version);
1782 }
1783
1784 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev,
1785                 struct device_attribute *attr, char *buf)
1786 {
1787         struct amdgpu_ras *con =
1788                 container_of(attr, struct amdgpu_ras, schema_attr);
1789         return sysfs_emit(buf, "schema: 0x%x\n", con->schema);
1790 }
1791
1792 static struct {
1793         enum ras_event_type type;
1794         const char *name;
1795 } dump_event[] = {
1796         {RAS_EVENT_TYPE_FATAL, "Fatal Error"},
1797         {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"},
1798         {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"},
1799 };
1800
1801 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev,
1802                                                  struct device_attribute *attr, char *buf)
1803 {
1804         struct amdgpu_ras *con =
1805                 container_of(attr, struct amdgpu_ras, event_state_attr);
1806         struct ras_event_manager *event_mgr = con->event_mgr;
1807         struct ras_event_state *event_state;
1808         int i, size = 0;
1809
1810         if (!event_mgr)
1811                 return -EINVAL;
1812
1813         size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno));
1814         for (i = 0; i < ARRAY_SIZE(dump_event); i++) {
1815                 event_state = &event_mgr->event_state[dump_event[i].type];
1816                 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n",
1817                                       dump_event[i].name,
1818                                       atomic64_read(&event_state->count),
1819                                       event_state->last_seqno);
1820         }
1821
1822         return (ssize_t)size;
1823 }
1824
1825 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1826 {
1827         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1828
1829         if (adev->dev->kobj.sd)
1830                 sysfs_remove_file_from_group(&adev->dev->kobj,
1831                                 &con->badpages_attr.attr,
1832                                 RAS_FS_NAME);
1833 }
1834
1835 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev)
1836 {
1837         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1838         struct attribute *attrs[] = {
1839                 &con->features_attr.attr,
1840                 &con->version_attr.attr,
1841                 &con->schema_attr.attr,
1842                 &con->event_state_attr.attr,
1843                 NULL
1844         };
1845         struct attribute_group group = {
1846                 .name = RAS_FS_NAME,
1847                 .attrs = attrs,
1848         };
1849
1850         if (adev->dev->kobj.sd)
1851                 sysfs_remove_group(&adev->dev->kobj, &group);
1852
1853         return 0;
1854 }
1855
1856 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
1857                 struct ras_common_if *head)
1858 {
1859         struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1860
1861         if (amdgpu_aca_is_enabled(adev))
1862                 return 0;
1863
1864         if (!obj || obj->attr_inuse)
1865                 return -EINVAL;
1866
1867         get_obj(obj);
1868
1869         snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1870                 "%s_err_count", head->name);
1871
1872         obj->sysfs_attr = (struct device_attribute){
1873                 .attr = {
1874                         .name = obj->fs_data.sysfs_name,
1875                         .mode = S_IRUGO,
1876                 },
1877                         .show = amdgpu_ras_sysfs_read,
1878         };
1879         sysfs_attr_init(&obj->sysfs_attr.attr);
1880
1881         if (sysfs_add_file_to_group(&adev->dev->kobj,
1882                                 &obj->sysfs_attr.attr,
1883                                 RAS_FS_NAME)) {
1884                 put_obj(obj);
1885                 return -EINVAL;
1886         }
1887
1888         obj->attr_inuse = 1;
1889
1890         return 0;
1891 }
1892
1893 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1894                 struct ras_common_if *head)
1895 {
1896         struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1897
1898         if (amdgpu_aca_is_enabled(adev))
1899                 return 0;
1900
1901         if (!obj || !obj->attr_inuse)
1902                 return -EINVAL;
1903
1904         if (adev->dev->kobj.sd)
1905                 sysfs_remove_file_from_group(&adev->dev->kobj,
1906                                 &obj->sysfs_attr.attr,
1907                                 RAS_FS_NAME);
1908         obj->attr_inuse = 0;
1909         put_obj(obj);
1910
1911         return 0;
1912 }
1913
1914 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1915 {
1916         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1917         struct ras_manager *obj, *tmp;
1918
1919         list_for_each_entry_safe(obj, tmp, &con->head, node) {
1920                 amdgpu_ras_sysfs_remove(adev, &obj->head);
1921         }
1922
1923         if (amdgpu_bad_page_threshold != 0)
1924                 amdgpu_ras_sysfs_remove_bad_page_node(adev);
1925
1926         amdgpu_ras_sysfs_remove_dev_attr_node(adev);
1927
1928         return 0;
1929 }
1930 /* sysfs end */
1931
1932 /**
1933  * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1934  *
1935  * Normally when there is an uncorrectable error, the driver will reset
1936  * the GPU to recover.  However, in the event of an unrecoverable error,
1937  * the driver provides an interface to reboot the system automatically
1938  * in that event.
1939  *
1940  * The following file in debugfs provides that interface:
1941  * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1942  *
1943  * Usage:
1944  *
1945  * .. code-block:: bash
1946  *
1947  *      echo true > .../ras/auto_reboot
1948  *
1949  */
1950 /* debugfs begin */
1951 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
1952 {
1953         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1954         struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control;
1955         struct drm_minor  *minor = adev_to_drm(adev)->primary;
1956         struct dentry     *dir;
1957
1958         dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1959         debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1960                             &amdgpu_ras_debugfs_ctrl_ops);
1961         debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1962                             &amdgpu_ras_debugfs_eeprom_ops);
1963         debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1964                            &con->bad_page_cnt_threshold);
1965         debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs);
1966         debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1967         debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1968         debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1969                             &amdgpu_ras_debugfs_eeprom_size_ops);
1970         con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1971                                                        S_IRUGO, dir, adev,
1972                                                        &amdgpu_ras_debugfs_eeprom_table_ops);
1973         amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
1974
1975         /*
1976          * After one uncorrectable error happens, usually GPU recovery will
1977          * be scheduled. But due to the known problem in GPU recovery failing
1978          * to bring GPU back, below interface provides one direct way to
1979          * user to reboot system automatically in such case within
1980          * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1981          * will never be called.
1982          */
1983         debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
1984
1985         /*
1986          * User could set this not to clean up hardware's error count register
1987          * of RAS IPs during ras recovery.
1988          */
1989         debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1990                             &con->disable_ras_err_cnt_harvest);
1991         return dir;
1992 }
1993
1994 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
1995                                       struct ras_fs_if *head,
1996                                       struct dentry *dir)
1997 {
1998         struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
1999
2000         if (!obj || !dir)
2001                 return;
2002
2003         get_obj(obj);
2004
2005         memcpy(obj->fs_data.debugfs_name,
2006                         head->debugfs_name,
2007                         sizeof(obj->fs_data.debugfs_name));
2008
2009         debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
2010                             obj, &amdgpu_ras_debugfs_ops);
2011 }
2012
2013 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev)
2014 {
2015         bool ret;
2016
2017         switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
2018         case IP_VERSION(13, 0, 6):
2019         case IP_VERSION(13, 0, 12):
2020         case IP_VERSION(13, 0, 14):
2021                 ret = true;
2022                 break;
2023         default:
2024                 ret = false;
2025                 break;
2026         }
2027
2028         return ret;
2029 }
2030
2031 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
2032 {
2033         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2034         struct dentry *dir;
2035         struct ras_manager *obj;
2036         struct ras_fs_if fs_info;
2037
2038         /*
2039          * it won't be called in resume path, no need to check
2040          * suspend and gpu reset status
2041          */
2042         if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
2043                 return;
2044
2045         dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
2046
2047         list_for_each_entry(obj, &con->head, node) {
2048                 if (amdgpu_ras_is_supported(adev, obj->head.block) &&
2049                         (obj->attr_inuse == 1)) {
2050                         sprintf(fs_info.debugfs_name, "%s_err_inject",
2051                                         get_ras_block_str(&obj->head));
2052                         fs_info.head = obj->head;
2053                         amdgpu_ras_debugfs_create(adev, &fs_info, dir);
2054                 }
2055         }
2056
2057         if (amdgpu_ras_aca_is_supported(adev)) {
2058                 if (amdgpu_aca_is_enabled(adev))
2059                         amdgpu_aca_smu_debugfs_init(adev, dir);
2060                 else
2061                         amdgpu_mca_smu_debugfs_init(adev, dir);
2062         }
2063 }
2064
2065 /* debugfs end */
2066
2067 /* ras fs */
2068 static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
2069                 amdgpu_ras_sysfs_badpages_read, NULL, 0);
2070 static DEVICE_ATTR(features, S_IRUGO,
2071                 amdgpu_ras_sysfs_features_read, NULL);
2072 static DEVICE_ATTR(version, 0444,
2073                 amdgpu_ras_sysfs_version_show, NULL);
2074 static DEVICE_ATTR(schema, 0444,
2075                 amdgpu_ras_sysfs_schema_show, NULL);
2076 static DEVICE_ATTR(event_state, 0444,
2077                    amdgpu_ras_sysfs_event_state_show, NULL);
2078 static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
2079 {
2080         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2081         struct attribute_group group = {
2082                 .name = RAS_FS_NAME,
2083         };
2084         struct attribute *attrs[] = {
2085                 &con->features_attr.attr,
2086                 &con->version_attr.attr,
2087                 &con->schema_attr.attr,
2088                 &con->event_state_attr.attr,
2089                 NULL
2090         };
2091         struct bin_attribute *bin_attrs[] = {
2092                 NULL,
2093                 NULL,
2094         };
2095         int r;
2096
2097         group.attrs = attrs;
2098
2099         /* add features entry */
2100         con->features_attr = dev_attr_features;
2101         sysfs_attr_init(attrs[0]);
2102
2103         /* add version entry */
2104         con->version_attr = dev_attr_version;
2105         sysfs_attr_init(attrs[1]);
2106
2107         /* add schema entry */
2108         con->schema_attr = dev_attr_schema;
2109         sysfs_attr_init(attrs[2]);
2110
2111         /* add event_state entry */
2112         con->event_state_attr = dev_attr_event_state;
2113         sysfs_attr_init(attrs[3]);
2114
2115         if (amdgpu_bad_page_threshold != 0) {
2116                 /* add bad_page_features entry */
2117                 bin_attr_gpu_vram_bad_pages.private = NULL;
2118                 con->badpages_attr = bin_attr_gpu_vram_bad_pages;
2119                 bin_attrs[0] = &con->badpages_attr;
2120                 group.bin_attrs = bin_attrs;
2121                 sysfs_bin_attr_init(bin_attrs[0]);
2122         }
2123
2124         r = sysfs_create_group(&adev->dev->kobj, &group);
2125         if (r)
2126                 dev_err(adev->dev, "Failed to create RAS sysfs group!");
2127
2128         return 0;
2129 }
2130
2131 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
2132 {
2133         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2134         struct ras_manager *con_obj, *ip_obj, *tmp;
2135
2136         if (IS_ENABLED(CONFIG_DEBUG_FS)) {
2137                 list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
2138                         ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
2139                         if (ip_obj)
2140                                 put_obj(ip_obj);
2141                 }
2142         }
2143
2144         amdgpu_ras_sysfs_remove_all(adev);
2145         return 0;
2146 }
2147 /* ras fs end */
2148
2149 /* ih begin */
2150
2151 /* For the hardware that cannot enable bif ring for both ras_controller_irq
2152  * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
2153  * register to check whether the interrupt is triggered or not, and properly
2154  * ack the interrupt if it is there
2155  */
2156 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
2157 {
2158         /* Fatal error events are handled on host side */
2159         if (amdgpu_sriov_vf(adev))
2160                 return;
2161         /**
2162          * If the current interrupt is caused by a non-fatal RAS error, skip
2163          * check for fatal error. For fatal errors, FED status of all devices
2164          * in XGMI hive gets set when the first device gets fatal error
2165          * interrupt. The error gets propagated to other devices as well, so
2166          * make sure to ack the interrupt regardless of FED status.
2167          */
2168         if (!amdgpu_ras_get_fed_status(adev) &&
2169             amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY))
2170                 return;
2171
2172         if (adev->nbio.ras &&
2173             adev->nbio.ras->handle_ras_controller_intr_no_bifring)
2174                 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
2175
2176         if (adev->nbio.ras &&
2177             adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
2178                 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
2179 }
2180
2181 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
2182                                 struct amdgpu_iv_entry *entry)
2183 {
2184         bool poison_stat = false;
2185         struct amdgpu_device *adev = obj->adev;
2186         struct amdgpu_ras_block_object *block_obj =
2187                 amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
2188         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2189         enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION;
2190         u64 event_id;
2191         int ret;
2192
2193         if (!block_obj || !con)
2194                 return;
2195
2196         ret = amdgpu_ras_mark_ras_event(adev, type);
2197         if (ret)
2198                 return;
2199
2200         amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block);
2201         /* both query_poison_status and handle_poison_consumption are optional,
2202          * but at least one of them should be implemented if we need poison
2203          * consumption handler
2204          */
2205         if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) {
2206                 poison_stat = block_obj->hw_ops->query_poison_status(adev);
2207                 if (!poison_stat) {
2208                         /* Not poison consumption interrupt, no need to handle it */
2209                         dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
2210                                         block_obj->ras_comm.name);
2211
2212                         return;
2213                 }
2214         }
2215
2216         amdgpu_umc_poison_handler(adev, obj->head.block, 0);
2217
2218         if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption)
2219                 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
2220
2221         /* gpu reset is fallback for failed and default cases.
2222          * For RMA case, amdgpu_umc_poison_handler will handle gpu reset.
2223          */
2224         if (poison_stat && !amdgpu_ras_is_rma(adev)) {
2225                 event_id = amdgpu_ras_acquire_event_id(adev, type);
2226                 RAS_EVENT_LOG(adev, event_id,
2227                               "GPU reset for %s RAS poison consumption is issued!\n",
2228                               block_obj->ras_comm.name);
2229                 amdgpu_ras_reset_gpu(adev);
2230         }
2231
2232         if (!poison_stat)
2233                 amdgpu_gfx_poison_consumption_handler(adev, entry);
2234 }
2235
2236 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
2237                                 struct amdgpu_iv_entry *entry)
2238 {
2239         struct amdgpu_device *adev = obj->adev;
2240         enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
2241         u64 event_id;
2242         int ret;
2243
2244         ret = amdgpu_ras_mark_ras_event(adev, type);
2245         if (ret)
2246                 return;
2247
2248         event_id = amdgpu_ras_acquire_event_id(adev, type);
2249         RAS_EVENT_LOG(adev, event_id, "Poison is created\n");
2250
2251         if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) {
2252                 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev);
2253
2254                 atomic_inc(&con->page_retirement_req_cnt);
2255                 atomic_inc(&con->poison_creation_count);
2256
2257                 wake_up(&con->page_retirement_wq);
2258         }
2259 }
2260
2261 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
2262                                 struct amdgpu_iv_entry *entry)
2263 {
2264         struct ras_ih_data *data = &obj->ih_data;
2265         struct ras_err_data err_data;
2266         int ret;
2267
2268         if (!data->cb)
2269                 return;
2270
2271         ret = amdgpu_ras_error_data_init(&err_data);
2272         if (ret)
2273                 return;
2274
2275         /* Let IP handle its data, maybe we need get the output
2276          * from the callback to update the error type/count, etc
2277          */
2278         amdgpu_ras_set_fed(obj->adev, true);
2279         ret = data->cb(obj->adev, &err_data, entry);
2280         /* ue will trigger an interrupt, and in that case
2281          * we need do a reset to recovery the whole system.
2282          * But leave IP do that recovery, here we just dispatch
2283          * the error.
2284          */
2285         if (ret == AMDGPU_RAS_SUCCESS) {
2286                 /* these counts could be left as 0 if
2287                  * some blocks do not count error number
2288                  */
2289                 obj->err_data.ue_count += err_data.ue_count;
2290                 obj->err_data.ce_count += err_data.ce_count;
2291                 obj->err_data.de_count += err_data.de_count;
2292         }
2293
2294         amdgpu_ras_error_data_fini(&err_data);
2295 }
2296
2297 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
2298 {
2299         struct ras_ih_data *data = &obj->ih_data;
2300         struct amdgpu_iv_entry entry;
2301
2302         while (data->rptr != data->wptr) {
2303                 rmb();
2304                 memcpy(&entry, &data->ring[data->rptr],
2305                                 data->element_size);
2306
2307                 wmb();
2308                 data->rptr = (data->aligned_element_size +
2309                                 data->rptr) % data->ring_size;
2310
2311                 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
2312                         if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2313                                 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
2314                         else
2315                                 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
2316                 } else {
2317                         if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2318                                 amdgpu_ras_interrupt_umc_handler(obj, &entry);
2319                         else
2320                                 dev_warn(obj->adev->dev,
2321                                         "No RAS interrupt handler for non-UMC block with poison disabled.\n");
2322                 }
2323         }
2324 }
2325
2326 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
2327 {
2328         struct ras_ih_data *data =
2329                 container_of(work, struct ras_ih_data, ih_work);
2330         struct ras_manager *obj =
2331                 container_of(data, struct ras_manager, ih_data);
2332
2333         amdgpu_ras_interrupt_handler(obj);
2334 }
2335
2336 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
2337                 struct ras_dispatch_if *info)
2338 {
2339         struct ras_manager *obj;
2340         struct ras_ih_data *data;
2341
2342         obj = amdgpu_ras_find_obj(adev, &info->head);
2343         if (!obj)
2344                 return -EINVAL;
2345
2346         data = &obj->ih_data;
2347
2348         if (data->inuse == 0)
2349                 return 0;
2350
2351         /* Might be overflow... */
2352         memcpy(&data->ring[data->wptr], info->entry,
2353                         data->element_size);
2354
2355         wmb();
2356         data->wptr = (data->aligned_element_size +
2357                         data->wptr) % data->ring_size;
2358
2359         schedule_work(&data->ih_work);
2360
2361         return 0;
2362 }
2363
2364 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
2365                 struct ras_common_if *head)
2366 {
2367         struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2368         struct ras_ih_data *data;
2369
2370         if (!obj)
2371                 return -EINVAL;
2372
2373         data = &obj->ih_data;
2374         if (data->inuse == 0)
2375                 return 0;
2376
2377         cancel_work_sync(&data->ih_work);
2378
2379         kfree(data->ring);
2380         memset(data, 0, sizeof(*data));
2381         put_obj(obj);
2382
2383         return 0;
2384 }
2385
2386 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
2387                 struct ras_common_if *head)
2388 {
2389         struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2390         struct ras_ih_data *data;
2391         struct amdgpu_ras_block_object *ras_obj;
2392
2393         if (!obj) {
2394                 /* in case we registe the IH before enable ras feature */
2395                 obj = amdgpu_ras_create_obj(adev, head);
2396                 if (!obj)
2397                         return -EINVAL;
2398         } else
2399                 get_obj(obj);
2400
2401         ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
2402
2403         data = &obj->ih_data;
2404         /* add the callback.etc */
2405         *data = (struct ras_ih_data) {
2406                 .inuse = 0,
2407                 .cb = ras_obj->ras_cb,
2408                 .element_size = sizeof(struct amdgpu_iv_entry),
2409                 .rptr = 0,
2410                 .wptr = 0,
2411         };
2412
2413         INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
2414
2415         data->aligned_element_size = ALIGN(data->element_size, 8);
2416         /* the ring can store 64 iv entries. */
2417         data->ring_size = 64 * data->aligned_element_size;
2418         data->ring = kmalloc(data->ring_size, GFP_KERNEL);
2419         if (!data->ring) {
2420                 put_obj(obj);
2421                 return -ENOMEM;
2422         }
2423
2424         /* IH is ready */
2425         data->inuse = 1;
2426
2427         return 0;
2428 }
2429
2430 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
2431 {
2432         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2433         struct ras_manager *obj, *tmp;
2434
2435         list_for_each_entry_safe(obj, tmp, &con->head, node) {
2436                 amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
2437         }
2438
2439         return 0;
2440 }
2441 /* ih end */
2442
2443 /* traversal all IPs except NBIO to query error counter */
2444 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type)
2445 {
2446         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2447         struct ras_manager *obj;
2448
2449         if (!adev->ras_enabled || !con)
2450                 return;
2451
2452         list_for_each_entry(obj, &con->head, node) {
2453                 struct ras_query_if info = {
2454                         .head = obj->head,
2455                 };
2456
2457                 /*
2458                  * PCIE_BIF IP has one different isr by ras controller
2459                  * interrupt, the specific ras counter query will be
2460                  * done in that isr. So skip such block from common
2461                  * sync flood interrupt isr calling.
2462                  */
2463                 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
2464                         continue;
2465
2466                 /*
2467                  * this is a workaround for aldebaran, skip send msg to
2468                  * smu to get ecc_info table due to smu handle get ecc
2469                  * info table failed temporarily.
2470                  * should be removed until smu fix handle ecc_info table.
2471                  */
2472                 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
2473                     (amdgpu_ip_version(adev, MP1_HWIP, 0) ==
2474                      IP_VERSION(13, 0, 2)))
2475                         continue;
2476
2477                 amdgpu_ras_query_error_status_with_event(adev, &info, type);
2478
2479                 if (amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2480                             IP_VERSION(11, 0, 2) &&
2481                     amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2482                             IP_VERSION(11, 0, 4) &&
2483                     amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2484                             IP_VERSION(13, 0, 0)) {
2485                         if (amdgpu_ras_reset_error_status(adev, info.head.block))
2486                                 dev_warn(adev->dev, "Failed to reset error counter and error status");
2487                 }
2488         }
2489 }
2490
2491 /* Parse RdRspStatus and WrRspStatus */
2492 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
2493                                           struct ras_query_if *info)
2494 {
2495         struct amdgpu_ras_block_object *block_obj;
2496         /*
2497          * Only two block need to query read/write
2498          * RspStatus at current state
2499          */
2500         if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
2501                 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
2502                 return;
2503
2504         block_obj = amdgpu_ras_get_ras_block(adev,
2505                                         info->head.block,
2506                                         info->head.sub_block_index);
2507
2508         if (!block_obj || !block_obj->hw_ops) {
2509                 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
2510                              get_ras_block_str(&info->head));
2511                 return;
2512         }
2513
2514         if (block_obj->hw_ops->query_ras_error_status)
2515                 block_obj->hw_ops->query_ras_error_status(adev);
2516
2517 }
2518
2519 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
2520 {
2521         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2522         struct ras_manager *obj;
2523
2524         if (!adev->ras_enabled || !con)
2525                 return;
2526
2527         list_for_each_entry(obj, &con->head, node) {
2528                 struct ras_query_if info = {
2529                         .head = obj->head,
2530                 };
2531
2532                 amdgpu_ras_error_status_query(adev, &info);
2533         }
2534 }
2535
2536 /* recovery begin */
2537
2538 /* return 0 on success.
2539  * caller need free bps.
2540  */
2541 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
2542                 struct ras_badpage **bps, unsigned int *count)
2543 {
2544         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2545         struct ras_err_handler_data *data;
2546         int i = 0;
2547         int ret = 0, status;
2548
2549         if (!con || !con->eh_data || !bps || !count)
2550                 return -EINVAL;
2551
2552         mutex_lock(&con->recovery_lock);
2553         data = con->eh_data;
2554         if (!data || data->count == 0) {
2555                 *bps = NULL;
2556                 ret = -EINVAL;
2557                 goto out;
2558         }
2559
2560         *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
2561         if (!*bps) {
2562                 ret = -ENOMEM;
2563                 goto out;
2564         }
2565
2566         for (; i < data->count; i++) {
2567                 (*bps)[i] = (struct ras_badpage){
2568                         .bp = data->bps[i].retired_page,
2569                         .size = AMDGPU_GPU_PAGE_SIZE,
2570                         .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
2571                 };
2572                 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
2573                                 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT);
2574                 if (status == -EBUSY)
2575                         (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
2576                 else if (status == -ENOENT)
2577                         (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
2578         }
2579
2580         *count = data->count;
2581 out:
2582         mutex_unlock(&con->recovery_lock);
2583         return ret;
2584 }
2585
2586 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev,
2587                                    struct amdgpu_hive_info *hive, bool status)
2588 {
2589         struct amdgpu_device *tmp_adev;
2590
2591         if (hive) {
2592                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
2593                         amdgpu_ras_set_fed(tmp_adev, status);
2594         } else {
2595                 amdgpu_ras_set_fed(adev, status);
2596         }
2597 }
2598
2599 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev)
2600 {
2601         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2602         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2603         int hive_ras_recovery = 0;
2604
2605         if (hive) {
2606                 hive_ras_recovery = atomic_read(&hive->ras_recovery);
2607                 amdgpu_put_xgmi_hive(hive);
2608         }
2609
2610         if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery))
2611                 return true;
2612
2613         return false;
2614 }
2615
2616 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev)
2617 {
2618         if (amdgpu_ras_intr_triggered())
2619                 return RAS_EVENT_TYPE_FATAL;
2620         else
2621                 return RAS_EVENT_TYPE_POISON_CONSUMPTION;
2622 }
2623
2624 static void amdgpu_ras_do_recovery(struct work_struct *work)
2625 {
2626         struct amdgpu_ras *ras =
2627                 container_of(work, struct amdgpu_ras, recovery_work);
2628         struct amdgpu_device *remote_adev = NULL;
2629         struct amdgpu_device *adev = ras->adev;
2630         struct list_head device_list, *device_list_handle =  NULL;
2631         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2632         enum ras_event_type type;
2633
2634         if (hive) {
2635                 atomic_set(&hive->ras_recovery, 1);
2636
2637                 /* If any device which is part of the hive received RAS fatal
2638                  * error interrupt, set fatal error status on all. This
2639                  * condition will need a recovery, and flag will be cleared
2640                  * as part of recovery.
2641                  */
2642                 list_for_each_entry(remote_adev, &hive->device_list,
2643                                     gmc.xgmi.head)
2644                         if (amdgpu_ras_get_fed_status(remote_adev)) {
2645                                 amdgpu_ras_set_fed_all(adev, hive, true);
2646                                 break;
2647                         }
2648         }
2649         if (!ras->disable_ras_err_cnt_harvest) {
2650
2651                 /* Build list of devices to query RAS related errors */
2652                 if  (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
2653                         device_list_handle = &hive->device_list;
2654                 } else {
2655                         INIT_LIST_HEAD(&device_list);
2656                         list_add_tail(&adev->gmc.xgmi.head, &device_list);
2657                         device_list_handle = &device_list;
2658                 }
2659
2660                 type = amdgpu_ras_get_fatal_error_event(adev);
2661                 list_for_each_entry(remote_adev,
2662                                 device_list_handle, gmc.xgmi.head) {
2663                         amdgpu_ras_query_err_status(remote_adev);
2664                         amdgpu_ras_log_on_err_counter(remote_adev, type);
2665                 }
2666
2667         }
2668
2669         if (amdgpu_device_should_recover_gpu(ras->adev)) {
2670                 struct amdgpu_reset_context reset_context;
2671                 memset(&reset_context, 0, sizeof(reset_context));
2672
2673                 reset_context.method = AMD_RESET_METHOD_NONE;
2674                 reset_context.reset_req_dev = adev;
2675                 reset_context.src = AMDGPU_RESET_SRC_RAS;
2676                 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
2677
2678                 /* Perform full reset in fatal error mode */
2679                 if (!amdgpu_ras_is_poison_mode_supported(ras->adev))
2680                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2681                 else {
2682                         clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2683
2684                         if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) {
2685                                 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET;
2686                                 reset_context.method = AMD_RESET_METHOD_MODE2;
2687                         }
2688
2689                         /* Fatal error occurs in poison mode, mode1 reset is used to
2690                          * recover gpu.
2691                          */
2692                         if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) {
2693                                 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET;
2694                                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2695
2696                                 psp_fatal_error_recovery_quirk(&adev->psp);
2697                         }
2698                 }
2699
2700                 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
2701         }
2702         atomic_set(&ras->in_recovery, 0);
2703         if (hive) {
2704                 atomic_set(&hive->ras_recovery, 0);
2705                 amdgpu_put_xgmi_hive(hive);
2706         }
2707 }
2708
2709 /* alloc/realloc bps array */
2710 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
2711                 struct ras_err_handler_data *data, int pages)
2712 {
2713         unsigned int old_space = data->count + data->space_left;
2714         unsigned int new_space = old_space + pages;
2715         unsigned int align_space = ALIGN(new_space, 512);
2716         void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
2717
2718         if (!bps) {
2719                 return -ENOMEM;
2720         }
2721
2722         if (data->bps) {
2723                 memcpy(bps, data->bps,
2724                                 data->count * sizeof(*data->bps));
2725                 kfree(data->bps);
2726         }
2727
2728         data->bps = bps;
2729         data->space_left += align_space - old_space;
2730         return 0;
2731 }
2732
2733 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev,
2734                         struct eeprom_table_record *bps,
2735                         struct ras_err_data *err_data)
2736 {
2737         struct ta_ras_query_address_input addr_in;
2738         uint32_t socket = 0;
2739         int ret = 0;
2740
2741         if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2742                 socket = adev->smuio.funcs->get_socket_id(adev);
2743
2744         /* reinit err_data */
2745         err_data->err_addr_cnt = 0;
2746         err_data->err_addr_len = adev->umc.retire_unit;
2747
2748         memset(&addr_in, 0, sizeof(addr_in));
2749         addr_in.ma.err_addr = bps->address;
2750         addr_in.ma.socket_id = socket;
2751         addr_in.ma.ch_inst = bps->mem_channel;
2752         /* tell RAS TA the node instance is not used */
2753         addr_in.ma.node_inst = TA_RAS_INV_NODE;
2754
2755         if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2756                 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data,
2757                                 &addr_in, NULL, false);
2758
2759         return ret;
2760 }
2761
2762 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev,
2763                         struct eeprom_table_record *bps,
2764                         struct ras_err_data *err_data)
2765 {
2766         struct ta_ras_query_address_input addr_in;
2767         uint32_t die_id, socket = 0;
2768
2769         if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2770                 socket = adev->smuio.funcs->get_socket_id(adev);
2771
2772         /* although die id is gotten from PA in nps1 mode, the id is
2773          * fitable for any nps mode
2774          */
2775         if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa)
2776                 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address,
2777                                         bps->retired_page << AMDGPU_GPU_PAGE_SHIFT);
2778         else
2779                 return -EINVAL;
2780
2781         /* reinit err_data */
2782         err_data->err_addr_cnt = 0;
2783         err_data->err_addr_len = adev->umc.retire_unit;
2784
2785         memset(&addr_in, 0, sizeof(addr_in));
2786         addr_in.ma.err_addr = bps->address;
2787         addr_in.ma.ch_inst = bps->mem_channel;
2788         addr_in.ma.umc_inst = bps->mcumc_id;
2789         addr_in.ma.node_inst = die_id;
2790         addr_in.ma.socket_id = socket;
2791
2792         if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2793                 return adev->umc.ras->convert_ras_err_addr(adev, err_data,
2794                                         &addr_in, NULL, false);
2795         else
2796                 return  -EINVAL;
2797 }
2798
2799 /* it deal with vram only. */
2800 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
2801                 struct eeprom_table_record *bps, int pages, bool from_rom)
2802 {
2803         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2804         struct ras_err_handler_data *data;
2805         struct ras_err_data err_data;
2806         struct eeprom_table_record *err_rec;
2807         struct amdgpu_ras_eeprom_control *control =
2808                         &adev->psp.ras_context.ras->eeprom_control;
2809         enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE;
2810         int ret = 0;
2811         uint32_t i, j, loop_cnt = 1;
2812         bool find_pages_per_pa = false;
2813
2814         if (!con || !con->eh_data || !bps || pages <= 0)
2815                 return 0;
2816
2817         if (from_rom) {
2818                 err_data.err_addr =
2819                         kcalloc(adev->umc.retire_unit,
2820                                 sizeof(struct eeprom_table_record), GFP_KERNEL);
2821                 if (!err_data.err_addr) {
2822                         dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n");
2823                         ret = -ENOMEM;
2824                         goto out;
2825                 }
2826
2827                 err_rec = err_data.err_addr;
2828                 loop_cnt = adev->umc.retire_unit;
2829                 if (adev->gmc.gmc_funcs->query_mem_partition_mode)
2830                         nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
2831         }
2832
2833         mutex_lock(&con->recovery_lock);
2834         data = con->eh_data;
2835         if (!data) {
2836                 /* Returning 0 as the absence of eh_data is acceptable */
2837                 goto free;
2838         }
2839
2840         for (i = 0; i < pages; i++) {
2841                 if (from_rom &&
2842                     control->rec_type == AMDGPU_RAS_EEPROM_REC_MCA) {
2843                         if (!find_pages_per_pa) {
2844                                 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[i], &err_data)) {
2845                                         if (!i && nps == AMDGPU_NPS1_PARTITION_MODE) {
2846                                                 /* may use old RAS TA, use PA to find pages in
2847                                                  * one row
2848                                                  */
2849                                                 if (amdgpu_umc_pages_in_a_row(adev, &err_data,
2850                                                                               bps[i].retired_page <<
2851                                                                               AMDGPU_GPU_PAGE_SHIFT)) {
2852                                                         ret = -EINVAL;
2853                                                         goto free;
2854                                                 } else {
2855                                                         find_pages_per_pa = true;
2856                                                 }
2857                                         } else {
2858                                                 /* unsupported cases */
2859                                                 ret = -EOPNOTSUPP;
2860                                                 goto free;
2861                                         }
2862                                 }
2863                         } else {
2864                                 if (amdgpu_umc_pages_in_a_row(adev, &err_data,
2865                                                 bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT)) {
2866                                         ret = -EINVAL;
2867                                         goto free;
2868                                 }
2869                         }
2870                 } else {
2871                         if (from_rom && !find_pages_per_pa) {
2872                                 if (bps[i].retired_page & UMC_CHANNEL_IDX_V2) {
2873                                         /* bad page in any NPS mode in eeprom */
2874                                         if (amdgpu_ras_mca2pa_by_idx(adev, &bps[i], &err_data)) {
2875                                                 ret = -EINVAL;
2876                                                 goto free;
2877                                         }
2878                                 } else {
2879                                         /* legacy bad page in eeprom, generated only in
2880                                          * NPS1 mode
2881                                          */
2882                                         if (amdgpu_ras_mca2pa(adev, &bps[i], &err_data)) {
2883                                                 /* old RAS TA or ASICs which don't support to
2884                                                  * convert addrss via mca address
2885                                                  */
2886                                                 if (!i && nps == AMDGPU_NPS1_PARTITION_MODE) {
2887                                                         find_pages_per_pa = true;
2888                                                         err_rec = &bps[i];
2889                                                         loop_cnt = 1;
2890                                                 } else {
2891                                                         /* non-nps1 mode, old RAS TA
2892                                                          * can't support it
2893                                                          */
2894                                                         ret = -EOPNOTSUPP;
2895                                                         goto free;
2896                                                 }
2897                                         }
2898                                 }
2899
2900                                 if (!find_pages_per_pa)
2901                                         i += (adev->umc.retire_unit - 1);
2902                         } else {
2903                                 err_rec = &bps[i];
2904                         }
2905                 }
2906
2907                 for (j = 0; j < loop_cnt; j++) {
2908                         if (amdgpu_ras_check_bad_page_unlock(con,
2909                                 err_rec[j].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2910                                 continue;
2911
2912                         if (!data->space_left &&
2913                             amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
2914                                 ret = -ENOMEM;
2915                                 goto free;
2916                         }
2917
2918                         amdgpu_ras_reserve_page(adev, err_rec[j].retired_page);
2919
2920                         memcpy(&data->bps[data->count], &(err_rec[j]),
2921                                         sizeof(struct eeprom_table_record));
2922                         data->count++;
2923                         data->space_left--;
2924                 }
2925         }
2926
2927 free:
2928         if (from_rom)
2929                 kfree(err_data.err_addr);
2930 out:
2931         mutex_unlock(&con->recovery_lock);
2932
2933         return ret;
2934 }
2935
2936 /*
2937  * write error record array to eeprom, the function should be
2938  * protected by recovery_lock
2939  * new_cnt: new added UE count, excluding reserved bad pages, can be NULL
2940  */
2941 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
2942                 unsigned long *new_cnt)
2943 {
2944         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2945         struct ras_err_handler_data *data;
2946         struct amdgpu_ras_eeprom_control *control;
2947         int save_count, unit_num, bad_page_num, i;
2948
2949         if (!con || !con->eh_data) {
2950                 if (new_cnt)
2951                         *new_cnt = 0;
2952
2953                 return 0;
2954         }
2955
2956         mutex_lock(&con->recovery_lock);
2957         control = &con->eeprom_control;
2958         data = con->eh_data;
2959         bad_page_num = control->ras_num_bad_pages;
2960         save_count = data->count - bad_page_num;
2961         mutex_unlock(&con->recovery_lock);
2962
2963         unit_num = save_count / adev->umc.retire_unit;
2964         if (new_cnt)
2965                 *new_cnt = unit_num;
2966
2967         /* only new entries are saved */
2968         if (save_count > 0) {
2969                 if (control->rec_type == AMDGPU_RAS_EEPROM_REC_PA) {
2970                         if (amdgpu_ras_eeprom_append(control,
2971                                                      &data->bps[control->ras_num_recs],
2972                                                      save_count)) {
2973                                 dev_err(adev->dev, "Failed to save EEPROM table data!");
2974                                 return -EIO;
2975                         }
2976                 } else {
2977                         for (i = 0; i < unit_num; i++) {
2978                                 if (amdgpu_ras_eeprom_append(control,
2979                                                 &data->bps[bad_page_num + i * adev->umc.retire_unit],
2980                                                 1)) {
2981                                         dev_err(adev->dev, "Failed to save EEPROM table data!");
2982                                         return -EIO;
2983                                 }
2984                         }
2985                 }
2986
2987                 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
2988         }
2989
2990         return 0;
2991 }
2992
2993 /*
2994  * read error record array in eeprom and reserve enough space for
2995  * storing new bad pages
2996  */
2997 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
2998 {
2999         struct amdgpu_ras_eeprom_control *control =
3000                 &adev->psp.ras_context.ras->eeprom_control;
3001         struct eeprom_table_record *bps;
3002         int ret;
3003
3004         /* no bad page record, skip eeprom access */
3005         if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
3006                 return 0;
3007
3008         bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
3009         if (!bps)
3010                 return -ENOMEM;
3011
3012         ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
3013         if (ret) {
3014                 dev_err(adev->dev, "Failed to load EEPROM table records!");
3015         } else {
3016                 if (control->ras_num_recs > 1 &&
3017                     adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {
3018                         if ((bps[0].address == bps[1].address) &&
3019                             (bps[0].mem_channel == bps[1].mem_channel))
3020                                 control->rec_type = AMDGPU_RAS_EEPROM_REC_PA;
3021                         else
3022                                 control->rec_type = AMDGPU_RAS_EEPROM_REC_MCA;
3023                 }
3024
3025                 ret = amdgpu_ras_eeprom_check(control);
3026                 if (ret)
3027                         goto out;
3028
3029                 /* HW not usable */
3030                 if (amdgpu_ras_is_rma(adev)) {
3031                         ret = -EHWPOISON;
3032                         goto out;
3033                 }
3034
3035                 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true);
3036         }
3037
3038 out:
3039         kfree(bps);
3040         return ret;
3041 }
3042
3043 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
3044                                 uint64_t addr)
3045 {
3046         struct ras_err_handler_data *data = con->eh_data;
3047         int i;
3048
3049         addr >>= AMDGPU_GPU_PAGE_SHIFT;
3050         for (i = 0; i < data->count; i++)
3051                 if (addr == data->bps[i].retired_page)
3052                         return true;
3053
3054         return false;
3055 }
3056
3057 /*
3058  * check if an address belongs to bad page
3059  *
3060  * Note: this check is only for umc block
3061  */
3062 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
3063                                 uint64_t addr)
3064 {
3065         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3066         bool ret = false;
3067
3068         if (!con || !con->eh_data)
3069                 return ret;
3070
3071         mutex_lock(&con->recovery_lock);
3072         ret = amdgpu_ras_check_bad_page_unlock(con, addr);
3073         mutex_unlock(&con->recovery_lock);
3074         return ret;
3075 }
3076
3077 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
3078                                           uint32_t max_count)
3079 {
3080         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3081
3082         /*
3083          * Justification of value bad_page_cnt_threshold in ras structure
3084          *
3085          * Generally, 0 <= amdgpu_bad_page_threshold <= max record length
3086          * in eeprom or amdgpu_bad_page_threshold == -2, introduce two
3087          * scenarios accordingly.
3088          *
3089          * Bad page retirement enablement:
3090          *    - If amdgpu_bad_page_threshold = -2,
3091          *      bad_page_cnt_threshold = typical value by formula.
3092          *
3093          *    - When the value from user is 0 < amdgpu_bad_page_threshold <
3094          *      max record length in eeprom, use it directly.
3095          *
3096          * Bad page retirement disablement:
3097          *    - If amdgpu_bad_page_threshold = 0, bad page retirement
3098          *      functionality is disabled, and bad_page_cnt_threshold will
3099          *      take no effect.
3100          */
3101
3102         if (amdgpu_bad_page_threshold < 0) {
3103                 u64 val = adev->gmc.mc_vram_size;
3104
3105                 do_div(val, RAS_BAD_PAGE_COVER);
3106                 con->bad_page_cnt_threshold = min(lower_32_bits(val),
3107                                                   max_count);
3108         } else {
3109                 con->bad_page_cnt_threshold = min_t(int, max_count,
3110                                                     amdgpu_bad_page_threshold);
3111         }
3112 }
3113
3114 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
3115                 enum amdgpu_ras_block block, uint16_t pasid,
3116                 pasid_notify pasid_fn, void *data, uint32_t reset)
3117 {
3118         int ret = 0;
3119         struct ras_poison_msg poison_msg;
3120         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3121
3122         memset(&poison_msg, 0, sizeof(poison_msg));
3123         poison_msg.block = block;
3124         poison_msg.pasid = pasid;
3125         poison_msg.reset = reset;
3126         poison_msg.pasid_fn = pasid_fn;
3127         poison_msg.data = data;
3128
3129         ret = kfifo_put(&con->poison_fifo, poison_msg);
3130         if (!ret) {
3131                 dev_err(adev->dev, "Poison message fifo is full!\n");
3132                 return -ENOSPC;
3133         }
3134
3135         return 0;
3136 }
3137
3138 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev,
3139                 struct ras_poison_msg *poison_msg)
3140 {
3141         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3142
3143         return kfifo_get(&con->poison_fifo, poison_msg);
3144 }
3145
3146 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log)
3147 {
3148         mutex_init(&ecc_log->lock);
3149
3150         INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL);
3151         ecc_log->de_queried_count = 0;
3152         ecc_log->prev_de_queried_count = 0;
3153 }
3154
3155 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log)
3156 {
3157         struct radix_tree_iter iter;
3158         void __rcu **slot;
3159         struct ras_ecc_err *ecc_err;
3160
3161         mutex_lock(&ecc_log->lock);
3162         radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) {
3163                 ecc_err = radix_tree_deref_slot(slot);
3164                 kfree(ecc_err->err_pages.pfn);
3165                 kfree(ecc_err);
3166                 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot);
3167         }
3168         mutex_unlock(&ecc_log->lock);
3169
3170         mutex_destroy(&ecc_log->lock);
3171         ecc_log->de_queried_count = 0;
3172         ecc_log->prev_de_queried_count = 0;
3173 }
3174
3175 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con,
3176                                 uint32_t delayed_ms)
3177 {
3178         int ret;
3179
3180         mutex_lock(&con->umc_ecc_log.lock);
3181         ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree,
3182                         UMC_ECC_NEW_DETECTED_TAG);
3183         mutex_unlock(&con->umc_ecc_log.lock);
3184
3185         if (ret)
3186                 schedule_delayed_work(&con->page_retirement_dwork,
3187                         msecs_to_jiffies(delayed_ms));
3188
3189         return ret ? true : false;
3190 }
3191
3192 static void amdgpu_ras_do_page_retirement(struct work_struct *work)
3193 {
3194         struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3195                                               page_retirement_dwork.work);
3196         struct amdgpu_device *adev = con->adev;
3197         struct ras_err_data err_data;
3198         unsigned long err_cnt;
3199
3200         /* If gpu reset is ongoing, delay retiring the bad pages */
3201         if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) {
3202                 amdgpu_ras_schedule_retirement_dwork(con,
3203                                 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3);
3204                 return;
3205         }
3206
3207         amdgpu_ras_error_data_init(&err_data);
3208
3209         amdgpu_umc_handle_bad_pages(adev, &err_data);
3210         err_cnt = err_data.err_addr_cnt;
3211
3212         amdgpu_ras_error_data_fini(&err_data);
3213
3214         if (err_cnt && amdgpu_ras_is_rma(adev))
3215                 amdgpu_ras_reset_gpu(adev);
3216
3217         amdgpu_ras_schedule_retirement_dwork(con,
3218                         AMDGPU_RAS_RETIRE_PAGE_INTERVAL);
3219 }
3220
3221 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev,
3222                                 uint32_t poison_creation_count)
3223 {
3224         int ret = 0;
3225         struct ras_ecc_log_info *ecc_log;
3226         struct ras_query_if info;
3227         uint32_t timeout = 0;
3228         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3229         uint64_t de_queried_count;
3230         uint32_t new_detect_count, total_detect_count;
3231         uint32_t need_query_count = poison_creation_count;
3232         bool query_data_timeout = false;
3233         enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
3234
3235         memset(&info, 0, sizeof(info));
3236         info.head.block = AMDGPU_RAS_BLOCK__UMC;
3237
3238         ecc_log = &ras->umc_ecc_log;
3239         total_detect_count = 0;
3240         do {
3241                 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type);
3242                 if (ret)
3243                         return ret;
3244
3245                 de_queried_count = ecc_log->de_queried_count;
3246                 if (de_queried_count > ecc_log->prev_de_queried_count) {
3247                         new_detect_count = de_queried_count - ecc_log->prev_de_queried_count;
3248                         ecc_log->prev_de_queried_count = de_queried_count;
3249                         timeout = 0;
3250                 } else {
3251                         new_detect_count = 0;
3252                 }
3253
3254                 if (new_detect_count) {
3255                         total_detect_count += new_detect_count;
3256                 } else {
3257                         if (!timeout && need_query_count)
3258                                 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC;
3259
3260                         if (timeout) {
3261                                 if (!--timeout) {
3262                                         query_data_timeout = true;
3263                                         break;
3264                                 }
3265                                 msleep(1);
3266                         }
3267                 }
3268         } while (total_detect_count < need_query_count);
3269
3270         if (query_data_timeout) {
3271                 dev_warn(adev->dev, "Can't find deferred error! count: %u\n",
3272                         (need_query_count - total_detect_count));
3273                 return -ENOENT;
3274         }
3275
3276         if (total_detect_count)
3277                 schedule_delayed_work(&ras->page_retirement_dwork, 0);
3278
3279         return 0;
3280 }
3281
3282 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev)
3283 {
3284         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3285         struct ras_poison_msg msg;
3286         int ret;
3287
3288         do {
3289                 ret = kfifo_get(&con->poison_fifo, &msg);
3290         } while (ret);
3291 }
3292
3293 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev,
3294                         uint32_t msg_count, uint32_t *gpu_reset)
3295 {
3296         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3297         uint32_t reset_flags = 0, reset = 0;
3298         struct ras_poison_msg msg;
3299         int ret, i;
3300
3301         kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
3302
3303         for (i = 0; i < msg_count; i++) {
3304                 ret = amdgpu_ras_get_poison_req(adev, &msg);
3305                 if (!ret)
3306                         continue;
3307
3308                 if (msg.pasid_fn)
3309                         msg.pasid_fn(adev, msg.pasid, msg.data);
3310
3311                 reset_flags |= msg.reset;
3312         }
3313
3314         /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */
3315         if (reset_flags && !amdgpu_ras_is_rma(adev)) {
3316                 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET)
3317                         reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET;
3318                 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET)
3319                         reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
3320                 else
3321                         reset = reset_flags;
3322
3323                 flush_delayed_work(&con->page_retirement_dwork);
3324
3325                 con->gpu_reset_flags |= reset;
3326                 amdgpu_ras_reset_gpu(adev);
3327
3328                 *gpu_reset = reset;
3329
3330                 /* Wait for gpu recovery to complete */
3331                 flush_work(&con->recovery_work);
3332         }
3333
3334         return 0;
3335 }
3336
3337 static int amdgpu_ras_page_retirement_thread(void *param)
3338 {
3339         struct amdgpu_device *adev = (struct amdgpu_device *)param;
3340         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3341         uint32_t poison_creation_count, msg_count;
3342         uint32_t gpu_reset;
3343         int ret;
3344
3345         while (!kthread_should_stop()) {
3346
3347                 wait_event_interruptible(con->page_retirement_wq,
3348                                 kthread_should_stop() ||
3349                                 atomic_read(&con->page_retirement_req_cnt));
3350
3351                 if (kthread_should_stop())
3352                         break;
3353
3354                 gpu_reset = 0;
3355
3356                 do {
3357                         poison_creation_count = atomic_read(&con->poison_creation_count);
3358                         ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count);
3359                         if (ret == -EIO)
3360                                 break;
3361
3362                         if (poison_creation_count) {
3363                                 atomic_sub(poison_creation_count, &con->poison_creation_count);
3364                                 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt);
3365                         }
3366                 } while (atomic_read(&con->poison_creation_count));
3367
3368                 if (ret != -EIO) {
3369                         msg_count = kfifo_len(&con->poison_fifo);
3370                         if (msg_count) {
3371                                 ret = amdgpu_ras_poison_consumption_handler(adev,
3372                                                 msg_count, &gpu_reset);
3373                                 if ((ret != -EIO) &&
3374                                     (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET))
3375                                         atomic_sub(msg_count, &con->page_retirement_req_cnt);
3376                         }
3377                 }
3378
3379                 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) {
3380                         /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */
3381                         /* Clear poison creation request */
3382                         atomic_set(&con->poison_creation_count, 0);
3383
3384                         /* Clear poison fifo */
3385                         amdgpu_ras_clear_poison_fifo(adev);
3386
3387                         /* Clear all poison requests */
3388                         atomic_set(&con->page_retirement_req_cnt, 0);
3389
3390                         if (ret == -EIO) {
3391                                 /* Wait for mode-1 reset to complete */
3392                                 down_read(&adev->reset_domain->sem);
3393                                 up_read(&adev->reset_domain->sem);
3394                         }
3395
3396                         /* Wake up work to save bad pages to eeprom */
3397                         schedule_delayed_work(&con->page_retirement_dwork, 0);
3398                 } else if (gpu_reset) {
3399                         /* gpu just completed mode-2 reset or other reset */
3400                         /* Clear poison consumption messages cached in fifo */
3401                         msg_count = kfifo_len(&con->poison_fifo);
3402                         if (msg_count) {
3403                                 amdgpu_ras_clear_poison_fifo(adev);
3404                                 atomic_sub(msg_count, &con->page_retirement_req_cnt);
3405                         }
3406
3407                         /* Wake up work to save bad pages to eeprom */
3408                         schedule_delayed_work(&con->page_retirement_dwork, 0);
3409                 }
3410         }
3411
3412         return 0;
3413 }
3414
3415 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev)
3416 {
3417         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3418         struct amdgpu_ras_eeprom_control *control;
3419         int ret;
3420
3421         if (!con || amdgpu_sriov_vf(adev))
3422                 return 0;
3423
3424         control = &con->eeprom_control;
3425         ret = amdgpu_ras_eeprom_init(control);
3426         if (ret)
3427                 return ret;
3428
3429         if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr)
3430                 control->rec_type = AMDGPU_RAS_EEPROM_REC_PA;
3431
3432         /* default status is MCA storage */
3433         if (control->ras_num_recs <= 1 &&
3434             adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
3435                 control->rec_type = AMDGPU_RAS_EEPROM_REC_MCA;
3436
3437         if (control->ras_num_recs) {
3438                 ret = amdgpu_ras_load_bad_pages(adev);
3439                 if (ret)
3440                         return ret;
3441
3442                 amdgpu_dpm_send_hbm_bad_pages_num(
3443                         adev, control->ras_num_bad_pages);
3444
3445                 if (con->update_channel_flag == true) {
3446                         amdgpu_dpm_send_hbm_bad_channel_flag(
3447                                 adev, control->bad_channel_bitmap);
3448                         con->update_channel_flag = false;
3449                 }
3450         }
3451
3452         return ret;
3453 }
3454
3455 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info)
3456 {
3457         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3458         struct ras_err_handler_data **data;
3459         u32  max_eeprom_records_count = 0;
3460         int ret;
3461
3462         if (!con || amdgpu_sriov_vf(adev))
3463                 return 0;
3464
3465         /* Allow access to RAS EEPROM via debugfs, when the ASIC
3466          * supports RAS and debugfs is enabled, but when
3467          * adev->ras_enabled is unset, i.e. when "ras_enable"
3468          * module parameter is set to 0.
3469          */
3470         con->adev = adev;
3471
3472         if (!adev->ras_enabled)
3473                 return 0;
3474
3475         data = &con->eh_data;
3476         *data = kzalloc(sizeof(**data), GFP_KERNEL);
3477         if (!*data) {
3478                 ret = -ENOMEM;
3479                 goto out;
3480         }
3481
3482         mutex_init(&con->recovery_lock);
3483         INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
3484         atomic_set(&con->in_recovery, 0);
3485         con->eeprom_control.bad_channel_bitmap = 0;
3486
3487         max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control);
3488         amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
3489
3490         if (init_bp_info) {
3491                 ret = amdgpu_ras_init_badpage_info(adev);
3492                 if (ret)
3493                         goto free;
3494         }
3495
3496         mutex_init(&con->page_rsv_lock);
3497         INIT_KFIFO(con->poison_fifo);
3498         mutex_init(&con->page_retirement_lock);
3499         init_waitqueue_head(&con->page_retirement_wq);
3500         atomic_set(&con->page_retirement_req_cnt, 0);
3501         atomic_set(&con->poison_creation_count, 0);
3502         con->page_retirement_thread =
3503                 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement");
3504         if (IS_ERR(con->page_retirement_thread)) {
3505                 con->page_retirement_thread = NULL;
3506                 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n");
3507         }
3508
3509         INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement);
3510         amdgpu_ras_ecc_log_init(&con->umc_ecc_log);
3511 #ifdef CONFIG_X86_MCE_AMD
3512         if ((adev->asic_type == CHIP_ALDEBARAN) &&
3513             (adev->gmc.xgmi.connected_to_cpu))
3514                 amdgpu_register_bad_pages_mca_notifier(adev);
3515 #endif
3516         return 0;
3517
3518 free:
3519         kfree((*data)->bps);
3520         kfree(*data);
3521         con->eh_data = NULL;
3522 out:
3523         dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
3524
3525         /*
3526          * Except error threshold exceeding case, other failure cases in this
3527          * function would not fail amdgpu driver init.
3528          */
3529         if (!amdgpu_ras_is_rma(adev))
3530                 ret = 0;
3531         else
3532                 ret = -EINVAL;
3533
3534         return ret;
3535 }
3536
3537 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
3538 {
3539         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3540         struct ras_err_handler_data *data = con->eh_data;
3541         int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES;
3542         bool ret;
3543
3544         /* recovery_init failed to init it, fini is useless */
3545         if (!data)
3546                 return 0;
3547
3548         /* Save all cached bad pages to eeprom */
3549         do {
3550                 flush_delayed_work(&con->page_retirement_dwork);
3551                 ret = amdgpu_ras_schedule_retirement_dwork(con, 0);
3552         } while (ret && max_flush_timeout--);
3553
3554         if (con->page_retirement_thread)
3555                 kthread_stop(con->page_retirement_thread);
3556
3557         atomic_set(&con->page_retirement_req_cnt, 0);
3558         atomic_set(&con->poison_creation_count, 0);
3559
3560         mutex_destroy(&con->page_rsv_lock);
3561
3562         cancel_work_sync(&con->recovery_work);
3563
3564         cancel_delayed_work_sync(&con->page_retirement_dwork);
3565
3566         amdgpu_ras_ecc_log_fini(&con->umc_ecc_log);
3567
3568         mutex_lock(&con->recovery_lock);
3569         con->eh_data = NULL;
3570         kfree(data->bps);
3571         kfree(data);
3572         mutex_unlock(&con->recovery_lock);
3573
3574         return 0;
3575 }
3576 /* recovery end */
3577
3578 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
3579 {
3580         if (amdgpu_sriov_vf(adev)) {
3581                 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3582                 case IP_VERSION(13, 0, 2):
3583                 case IP_VERSION(13, 0, 6):
3584                 case IP_VERSION(13, 0, 12):
3585                 case IP_VERSION(13, 0, 14):
3586                         return true;
3587                 default:
3588                         return false;
3589                 }
3590         }
3591
3592         if (adev->asic_type == CHIP_IP_DISCOVERY) {
3593                 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3594                 case IP_VERSION(13, 0, 0):
3595                 case IP_VERSION(13, 0, 6):
3596                 case IP_VERSION(13, 0, 10):
3597                 case IP_VERSION(13, 0, 12):
3598                 case IP_VERSION(13, 0, 14):
3599                 case IP_VERSION(14, 0, 3):
3600                         return true;
3601                 default:
3602                         return false;
3603                 }
3604         }
3605
3606         return adev->asic_type == CHIP_VEGA10 ||
3607                 adev->asic_type == CHIP_VEGA20 ||
3608                 adev->asic_type == CHIP_ARCTURUS ||
3609                 adev->asic_type == CHIP_ALDEBARAN ||
3610                 adev->asic_type == CHIP_SIENNA_CICHLID;
3611 }
3612
3613 /*
3614  * this is workaround for vega20 workstation sku,
3615  * force enable gfx ras, ignore vbios gfx ras flag
3616  * due to GC EDC can not write
3617  */
3618 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
3619 {
3620         struct atom_context *ctx = adev->mode_info.atom_context;
3621
3622         if (!ctx)
3623                 return;
3624
3625         if (strnstr(ctx->vbios_pn, "D16406",
3626                     sizeof(ctx->vbios_pn)) ||
3627                 strnstr(ctx->vbios_pn, "D36002",
3628                         sizeof(ctx->vbios_pn)))
3629                 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
3630 }
3631
3632 /* Query ras capablity via atomfirmware interface */
3633 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev)
3634 {
3635         /* mem_ecc cap */
3636         if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
3637                 dev_info(adev->dev, "MEM ECC is active.\n");
3638                 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
3639                                          1 << AMDGPU_RAS_BLOCK__DF);
3640         } else {
3641                 dev_info(adev->dev, "MEM ECC is not presented.\n");
3642         }
3643
3644         /* sram_ecc cap */
3645         if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
3646                 dev_info(adev->dev, "SRAM ECC is active.\n");
3647                 if (!amdgpu_sriov_vf(adev))
3648                         adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
3649                                                   1 << AMDGPU_RAS_BLOCK__DF);
3650                 else
3651                         adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
3652                                                  1 << AMDGPU_RAS_BLOCK__SDMA |
3653                                                  1 << AMDGPU_RAS_BLOCK__GFX);
3654
3655                 /*
3656                  * VCN/JPEG RAS can be supported on both bare metal and
3657                  * SRIOV environment
3658                  */
3659                 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) ||
3660                     amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) ||
3661                     amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3))
3662                         adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
3663                                                  1 << AMDGPU_RAS_BLOCK__JPEG);
3664                 else
3665                         adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
3666                                                   1 << AMDGPU_RAS_BLOCK__JPEG);
3667
3668                 /*
3669                  * XGMI RAS is not supported if xgmi num physical nodes
3670                  * is zero
3671                  */
3672                 if (!adev->gmc.xgmi.num_physical_nodes)
3673                         adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL);
3674         } else {
3675                 dev_info(adev->dev, "SRAM ECC is not presented.\n");
3676         }
3677 }
3678
3679 /* Query poison mode from umc/df IP callbacks */
3680 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev)
3681 {
3682         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3683         bool df_poison, umc_poison;
3684
3685         /* poison setting is useless on SRIOV guest */
3686         if (amdgpu_sriov_vf(adev) || !con)
3687                 return;
3688
3689         /* Init poison supported flag, the default value is false */
3690         if (adev->gmc.xgmi.connected_to_cpu ||
3691             adev->gmc.is_app_apu) {
3692                 /* enabled by default when GPU is connected to CPU */
3693                 con->poison_supported = true;
3694         } else if (adev->df.funcs &&
3695             adev->df.funcs->query_ras_poison_mode &&
3696             adev->umc.ras &&
3697             adev->umc.ras->query_ras_poison_mode) {
3698                 df_poison =
3699                         adev->df.funcs->query_ras_poison_mode(adev);
3700                 umc_poison =
3701                         adev->umc.ras->query_ras_poison_mode(adev);
3702
3703                 /* Only poison is set in both DF and UMC, we can support it */
3704                 if (df_poison && umc_poison)
3705                         con->poison_supported = true;
3706                 else if (df_poison != umc_poison)
3707                         dev_warn(adev->dev,
3708                                 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
3709                                 df_poison, umc_poison);
3710         }
3711 }
3712
3713 /*
3714  * check hardware's ras ability which will be saved in hw_supported.
3715  * if hardware does not support ras, we can skip some ras initializtion and
3716  * forbid some ras operations from IP.
3717  * if software itself, say boot parameter, limit the ras ability. We still
3718  * need allow IP do some limited operations, like disable. In such case,
3719  * we have to initialize ras as normal. but need check if operation is
3720  * allowed or not in each function.
3721  */
3722 static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
3723 {
3724         adev->ras_hw_enabled = adev->ras_enabled = 0;
3725
3726         if (!amdgpu_ras_asic_supported(adev))
3727                 return;
3728
3729         if (amdgpu_sriov_vf(adev)) {
3730                 if (amdgpu_virt_get_ras_capability(adev))
3731                         goto init_ras_enabled_flag;
3732         }
3733
3734         /* query ras capability from psp */
3735         if (amdgpu_psp_get_ras_capability(&adev->psp))
3736                 goto init_ras_enabled_flag;
3737
3738         /* query ras capablity from bios */
3739         if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
3740                 amdgpu_ras_query_ras_capablity_from_vbios(adev);
3741         } else {
3742                 /* driver only manages a few IP blocks RAS feature
3743                  * when GPU is connected cpu through XGMI */
3744                 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
3745                                            1 << AMDGPU_RAS_BLOCK__SDMA |
3746                                            1 << AMDGPU_RAS_BLOCK__MMHUB);
3747         }
3748
3749         /* apply asic specific settings (vega20 only for now) */
3750         amdgpu_ras_get_quirks(adev);
3751
3752         /* query poison mode from umc/df ip callback */
3753         amdgpu_ras_query_poison_mode(adev);
3754
3755 init_ras_enabled_flag:
3756         /* hw_supported needs to be aligned with RAS block mask. */
3757         adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
3758
3759         adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
3760                 adev->ras_hw_enabled & amdgpu_ras_mask;
3761
3762         /* aca is disabled by default */
3763         adev->aca.is_enabled = false;
3764
3765         /* bad page feature is not applicable to specific app platform */
3766         if (adev->gmc.is_app_apu &&
3767             amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0))
3768                 amdgpu_bad_page_threshold = 0;
3769 }
3770
3771 static void amdgpu_ras_counte_dw(struct work_struct *work)
3772 {
3773         struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3774                                               ras_counte_delay_work.work);
3775         struct amdgpu_device *adev = con->adev;
3776         struct drm_device *dev = adev_to_drm(adev);
3777         unsigned long ce_count, ue_count;
3778         int res;
3779
3780         res = pm_runtime_get_sync(dev->dev);
3781         if (res < 0)
3782                 goto Out;
3783
3784         /* Cache new values.
3785          */
3786         if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) {
3787                 atomic_set(&con->ras_ce_count, ce_count);
3788                 atomic_set(&con->ras_ue_count, ue_count);
3789         }
3790
3791         pm_runtime_mark_last_busy(dev->dev);
3792 Out:
3793         pm_runtime_put_autosuspend(dev->dev);
3794 }
3795
3796 static int amdgpu_get_ras_schema(struct amdgpu_device *adev)
3797 {
3798         return  amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 |
3799                         AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE |
3800                         AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE |
3801                         AMDGPU_RAS_ERROR__PARITY;
3802 }
3803
3804 static void ras_event_mgr_init(struct ras_event_manager *mgr)
3805 {
3806         struct ras_event_state *event_state;
3807         int i;
3808
3809         memset(mgr, 0, sizeof(*mgr));
3810         atomic64_set(&mgr->seqno, 0);
3811
3812         for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) {
3813                 event_state = &mgr->event_state[i];
3814                 event_state->last_seqno = RAS_EVENT_INVALID_ID;
3815                 atomic64_set(&event_state->count, 0);
3816         }
3817 }
3818
3819 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev)
3820 {
3821         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3822         struct amdgpu_hive_info *hive;
3823
3824         if (!ras)
3825                 return;
3826
3827         hive = amdgpu_get_xgmi_hive(adev);
3828         ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr;
3829
3830         /* init event manager with node 0 on xgmi system */
3831         if (!amdgpu_reset_in_recovery(adev)) {
3832                 if (!hive || adev->gmc.xgmi.node_id == 0)
3833                         ras_event_mgr_init(ras->event_mgr);
3834         }
3835
3836         if (hive)
3837                 amdgpu_put_xgmi_hive(hive);
3838 }
3839
3840 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev)
3841 {
3842         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3843
3844         if (!con || (adev->flags & AMD_IS_APU))
3845                 return;
3846
3847         switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3848         case IP_VERSION(13, 0, 2):
3849         case IP_VERSION(13, 0, 6):
3850         case IP_VERSION(13, 0, 12):
3851         case IP_VERSION(13, 0, 14):
3852                 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE;
3853                 break;
3854         default:
3855                 break;
3856         }
3857 }
3858
3859 int amdgpu_ras_init(struct amdgpu_device *adev)
3860 {
3861         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3862         int r;
3863
3864         if (con)
3865                 return 0;
3866
3867         con = kzalloc(sizeof(*con) +
3868                         sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
3869                         sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
3870                         GFP_KERNEL);
3871         if (!con)
3872                 return -ENOMEM;
3873
3874         con->adev = adev;
3875         INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
3876         atomic_set(&con->ras_ce_count, 0);
3877         atomic_set(&con->ras_ue_count, 0);
3878
3879         con->objs = (struct ras_manager *)(con + 1);
3880
3881         amdgpu_ras_set_context(adev, con);
3882
3883         amdgpu_ras_check_supported(adev);
3884
3885         if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
3886                 /* set gfx block ras context feature for VEGA20 Gaming
3887                  * send ras disable cmd to ras ta during ras late init.
3888                  */
3889                 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
3890                         con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
3891
3892                         return 0;
3893                 }
3894
3895                 r = 0;
3896                 goto release_con;
3897         }
3898
3899         con->update_channel_flag = false;
3900         con->features = 0;
3901         con->schema = 0;
3902         INIT_LIST_HEAD(&con->head);
3903         /* Might need get this flag from vbios. */
3904         con->flags = RAS_DEFAULT_FLAGS;
3905
3906         /* initialize nbio ras function ahead of any other
3907          * ras functions so hardware fatal error interrupt
3908          * can be enabled as early as possible */
3909         switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) {
3910         case IP_VERSION(7, 4, 0):
3911         case IP_VERSION(7, 4, 1):
3912         case IP_VERSION(7, 4, 4):
3913                 if (!adev->gmc.xgmi.connected_to_cpu)
3914                         adev->nbio.ras = &nbio_v7_4_ras;
3915                 break;
3916         case IP_VERSION(4, 3, 0):
3917                 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
3918                         /* unlike other generation of nbio ras,
3919                          * nbio v4_3 only support fatal error interrupt
3920                          * to inform software that DF is freezed due to
3921                          * system fatal error event. driver should not
3922                          * enable nbio ras in such case. Instead,
3923                          * check DF RAS */
3924                         adev->nbio.ras = &nbio_v4_3_ras;
3925                 break;
3926         case IP_VERSION(6, 3, 1):
3927                 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
3928                         /* unlike other generation of nbio ras,
3929                          * nbif v6_3_1 only support fatal error interrupt
3930                          * to inform software that DF is freezed due to
3931                          * system fatal error event. driver should not
3932                          * enable nbio ras in such case. Instead,
3933                          * check DF RAS
3934                          */
3935                         adev->nbio.ras = &nbif_v6_3_1_ras;
3936                 break;
3937         case IP_VERSION(7, 9, 0):
3938         case IP_VERSION(7, 9, 1):
3939                 if (!adev->gmc.is_app_apu)
3940                         adev->nbio.ras = &nbio_v7_9_ras;
3941                 break;
3942         default:
3943                 /* nbio ras is not available */
3944                 break;
3945         }
3946
3947         /* nbio ras block needs to be enabled ahead of other ras blocks
3948          * to handle fatal error */
3949         r = amdgpu_nbio_ras_sw_init(adev);
3950         if (r)
3951                 return r;
3952
3953         if (adev->nbio.ras &&
3954             adev->nbio.ras->init_ras_controller_interrupt) {
3955                 r = adev->nbio.ras->init_ras_controller_interrupt(adev);
3956                 if (r)
3957                         goto release_con;
3958         }
3959
3960         if (adev->nbio.ras &&
3961             adev->nbio.ras->init_ras_err_event_athub_interrupt) {
3962                 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
3963                 if (r)
3964                         goto release_con;
3965         }
3966
3967         /* Packed socket_id to ras feature mask bits[31:29] */
3968         if (adev->smuio.funcs &&
3969             adev->smuio.funcs->get_socket_id)
3970                 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) <<
3971                                         AMDGPU_RAS_FEATURES_SOCKETID_SHIFT);
3972
3973         /* Get RAS schema for particular SOC */
3974         con->schema = amdgpu_get_ras_schema(adev);
3975
3976         amdgpu_ras_init_reserved_vram_size(adev);
3977
3978         if (amdgpu_ras_fs_init(adev)) {
3979                 r = -EINVAL;
3980                 goto release_con;
3981         }
3982
3983         if (amdgpu_ras_aca_is_supported(adev)) {
3984                 if (amdgpu_aca_is_enabled(adev))
3985                         r = amdgpu_aca_init(adev);
3986                 else
3987                         r = amdgpu_mca_init(adev);
3988                 if (r)
3989                         goto release_con;
3990         }
3991
3992         dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
3993                  "hardware ability[%x] ras_mask[%x]\n",
3994                  adev->ras_hw_enabled, adev->ras_enabled);
3995
3996         return 0;
3997 release_con:
3998         amdgpu_ras_set_context(adev, NULL);
3999         kfree(con);
4000
4001         return r;
4002 }
4003
4004 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
4005 {
4006         if (adev->gmc.xgmi.connected_to_cpu ||
4007             adev->gmc.is_app_apu)
4008                 return 1;
4009         return 0;
4010 }
4011
4012 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
4013                                         struct ras_common_if *ras_block)
4014 {
4015         struct ras_query_if info = {
4016                 .head = *ras_block,
4017         };
4018
4019         if (!amdgpu_persistent_edc_harvesting_supported(adev))
4020                 return 0;
4021
4022         if (amdgpu_ras_query_error_status(adev, &info) != 0)
4023                 DRM_WARN("RAS init harvest failure");
4024
4025         if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
4026                 DRM_WARN("RAS init harvest reset failure");
4027
4028         return 0;
4029 }
4030
4031 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
4032 {
4033        struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4034
4035        if (!con)
4036                return false;
4037
4038        return con->poison_supported;
4039 }
4040
4041 /* helper function to handle common stuff in ip late init phase */
4042 int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
4043                          struct ras_common_if *ras_block)
4044 {
4045         struct amdgpu_ras_block_object *ras_obj = NULL;
4046         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4047         struct ras_query_if *query_info;
4048         unsigned long ue_count, ce_count;
4049         int r;
4050
4051         /* disable RAS feature per IP block if it is not supported */
4052         if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
4053                 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
4054                 return 0;
4055         }
4056
4057         r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
4058         if (r) {
4059                 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) {
4060                         /* in resume phase, if fail to enable ras,
4061                          * clean up all ras fs nodes, and disable ras */
4062                         goto cleanup;
4063                 } else
4064                         return r;
4065         }
4066
4067         /* check for errors on warm reset edc persisant supported ASIC */
4068         amdgpu_persistent_edc_harvesting(adev, ras_block);
4069
4070         /* in resume phase, no need to create ras fs node */
4071         if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
4072                 return 0;
4073
4074         ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4075         if (ras_obj->ras_cb || (ras_obj->hw_ops &&
4076             (ras_obj->hw_ops->query_poison_status ||
4077             ras_obj->hw_ops->handle_poison_consumption))) {
4078                 r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
4079                 if (r)
4080                         goto cleanup;
4081         }
4082
4083         if (ras_obj->hw_ops &&
4084             (ras_obj->hw_ops->query_ras_error_count ||
4085              ras_obj->hw_ops->query_ras_error_status)) {
4086                 r = amdgpu_ras_sysfs_create(adev, ras_block);
4087                 if (r)
4088                         goto interrupt;
4089
4090                 /* Those are the cached values at init.
4091                  */
4092                 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL);
4093                 if (!query_info)
4094                         return -ENOMEM;
4095                 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if));
4096
4097                 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) {
4098                         atomic_set(&con->ras_ce_count, ce_count);
4099                         atomic_set(&con->ras_ue_count, ue_count);
4100                 }
4101
4102                 kfree(query_info);
4103         }
4104
4105         return 0;
4106
4107 interrupt:
4108         if (ras_obj->ras_cb)
4109                 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
4110 cleanup:
4111         amdgpu_ras_feature_enable(adev, ras_block, 0);
4112         return r;
4113 }
4114
4115 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
4116                          struct ras_common_if *ras_block)
4117 {
4118         return amdgpu_ras_block_late_init(adev, ras_block);
4119 }
4120
4121 /* helper function to remove ras fs node and interrupt handler */
4122 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
4123                           struct ras_common_if *ras_block)
4124 {
4125         struct amdgpu_ras_block_object *ras_obj;
4126         if (!ras_block)
4127                 return;
4128
4129         amdgpu_ras_sysfs_remove(adev, ras_block);
4130
4131         ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4132         if (ras_obj->ras_cb)
4133                 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
4134 }
4135
4136 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
4137                           struct ras_common_if *ras_block)
4138 {
4139         return amdgpu_ras_block_late_fini(adev, ras_block);
4140 }
4141
4142 /* do some init work after IP late init as dependence.
4143  * and it runs in resume/gpu reset/booting up cases.
4144  */
4145 void amdgpu_ras_resume(struct amdgpu_device *adev)
4146 {
4147         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4148         struct ras_manager *obj, *tmp;
4149
4150         if (!adev->ras_enabled || !con) {
4151                 /* clean ras context for VEGA20 Gaming after send ras disable cmd */
4152                 amdgpu_release_ras_context(adev);
4153
4154                 return;
4155         }
4156
4157         if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
4158                 /* Set up all other IPs which are not implemented. There is a
4159                  * tricky thing that IP's actual ras error type should be
4160                  * MULTI_UNCORRECTABLE, but as driver does not handle it, so
4161                  * ERROR_NONE make sense anyway.
4162                  */
4163                 amdgpu_ras_enable_all_features(adev, 1);
4164
4165                 /* We enable ras on all hw_supported block, but as boot
4166                  * parameter might disable some of them and one or more IP has
4167                  * not implemented yet. So we disable them on behalf.
4168                  */
4169                 list_for_each_entry_safe(obj, tmp, &con->head, node) {
4170                         if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
4171                                 amdgpu_ras_feature_enable(adev, &obj->head, 0);
4172                                 /* there should be no any reference. */
4173                                 WARN_ON(alive_obj(obj));
4174                         }
4175                 }
4176         }
4177 }
4178
4179 void amdgpu_ras_suspend(struct amdgpu_device *adev)
4180 {
4181         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4182
4183         if (!adev->ras_enabled || !con)
4184                 return;
4185
4186         amdgpu_ras_disable_all_features(adev, 0);
4187         /* Make sure all ras objects are disabled. */
4188         if (AMDGPU_RAS_GET_FEATURES(con->features))
4189                 amdgpu_ras_disable_all_features(adev, 1);
4190 }
4191
4192 int amdgpu_ras_late_init(struct amdgpu_device *adev)
4193 {
4194         struct amdgpu_ras_block_list *node, *tmp;
4195         struct amdgpu_ras_block_object *obj;
4196         int r;
4197
4198         amdgpu_ras_event_mgr_init(adev);
4199
4200         if (amdgpu_ras_aca_is_supported(adev)) {
4201                 if (amdgpu_reset_in_recovery(adev)) {
4202                         if (amdgpu_aca_is_enabled(adev))
4203                                 r = amdgpu_aca_reset(adev);
4204                         else
4205                                 r = amdgpu_mca_reset(adev);
4206                         if (r)
4207                                 return r;
4208                 }
4209
4210                 if (!amdgpu_sriov_vf(adev)) {
4211                         if (amdgpu_aca_is_enabled(adev))
4212                                 amdgpu_ras_set_aca_debug_mode(adev, false);
4213                         else
4214                                 amdgpu_ras_set_mca_debug_mode(adev, false);
4215                 }
4216         }
4217
4218         /* Guest side doesn't need init ras feature */
4219         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev))
4220                 return 0;
4221
4222         list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
4223                 obj = node->ras_obj;
4224                 if (!obj) {
4225                         dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
4226                         continue;
4227                 }
4228
4229                 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block))
4230                         continue;
4231
4232                 if (obj->ras_late_init) {
4233                         r = obj->ras_late_init(adev, &obj->ras_comm);
4234                         if (r) {
4235                                 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
4236                                         obj->ras_comm.name, r);
4237                                 return r;
4238                         }
4239                 } else
4240                         amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
4241         }
4242
4243         return 0;
4244 }
4245
4246 /* do some fini work before IP fini as dependence */
4247 int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
4248 {
4249         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4250
4251         if (!adev->ras_enabled || !con)
4252                 return 0;
4253
4254
4255         /* Need disable ras on all IPs here before ip [hw/sw]fini */
4256         if (AMDGPU_RAS_GET_FEATURES(con->features))
4257                 amdgpu_ras_disable_all_features(adev, 0);
4258         amdgpu_ras_recovery_fini(adev);
4259         return 0;
4260 }
4261
4262 int amdgpu_ras_fini(struct amdgpu_device *adev)
4263 {
4264         struct amdgpu_ras_block_list *ras_node, *tmp;
4265         struct amdgpu_ras_block_object *obj = NULL;
4266         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4267
4268         if (!adev->ras_enabled || !con)
4269                 return 0;
4270
4271         list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
4272                 if (ras_node->ras_obj) {
4273                         obj = ras_node->ras_obj;
4274                         if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
4275                             obj->ras_fini)
4276                                 obj->ras_fini(adev, &obj->ras_comm);
4277                         else
4278                                 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
4279                 }
4280
4281                 /* Clear ras blocks from ras_list and free ras block list node */
4282                 list_del(&ras_node->node);
4283                 kfree(ras_node);
4284         }
4285
4286         amdgpu_ras_fs_fini(adev);
4287         amdgpu_ras_interrupt_remove_all(adev);
4288
4289         if (amdgpu_ras_aca_is_supported(adev)) {
4290                 if (amdgpu_aca_is_enabled(adev))
4291                         amdgpu_aca_fini(adev);
4292                 else
4293                         amdgpu_mca_fini(adev);
4294         }
4295
4296         WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared");
4297
4298         if (AMDGPU_RAS_GET_FEATURES(con->features))
4299                 amdgpu_ras_disable_all_features(adev, 0);
4300
4301         cancel_delayed_work_sync(&con->ras_counte_delay_work);
4302
4303         amdgpu_ras_set_context(adev, NULL);
4304         kfree(con);
4305
4306         return 0;
4307 }
4308
4309 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev)
4310 {
4311         struct amdgpu_ras *ras;
4312
4313         ras = amdgpu_ras_get_context(adev);
4314         if (!ras)
4315                 return false;
4316
4317         return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4318 }
4319
4320 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status)
4321 {
4322         struct amdgpu_ras *ras;
4323
4324         ras = amdgpu_ras_get_context(adev);
4325         if (ras) {
4326                 if (status)
4327                         set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4328                 else
4329                         clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4330         }
4331 }
4332
4333 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev)
4334 {
4335         struct amdgpu_ras *ras;
4336
4337         ras = amdgpu_ras_get_context(adev);
4338         if (ras)
4339                 ras->ras_err_state = 0;
4340 }
4341
4342 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev,
4343                                enum amdgpu_ras_block block)
4344 {
4345         struct amdgpu_ras *ras;
4346
4347         ras = amdgpu_ras_get_context(adev);
4348         if (ras)
4349                 set_bit(block, &ras->ras_err_state);
4350 }
4351
4352 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block)
4353 {
4354         struct amdgpu_ras *ras;
4355
4356         ras = amdgpu_ras_get_context(adev);
4357         if (ras) {
4358                 if (block == AMDGPU_RAS_BLOCK__ANY)
4359                         return (ras->ras_err_state != 0);
4360                 else
4361                         return test_bit(block, &ras->ras_err_state) ||
4362                                test_bit(AMDGPU_RAS_BLOCK__LAST,
4363                                         &ras->ras_err_state);
4364         }
4365
4366         return false;
4367 }
4368
4369 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev)
4370 {
4371         struct amdgpu_ras *ras;
4372
4373         ras = amdgpu_ras_get_context(adev);
4374         if (!ras)
4375                 return NULL;
4376
4377         return ras->event_mgr;
4378 }
4379
4380 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type,
4381                                      const void *caller)
4382 {
4383         struct ras_event_manager *event_mgr;
4384         struct ras_event_state *event_state;
4385         int ret = 0;
4386
4387         if (type >= RAS_EVENT_TYPE_COUNT) {
4388                 ret = -EINVAL;
4389                 goto out;
4390         }
4391
4392         event_mgr = __get_ras_event_mgr(adev);
4393         if (!event_mgr) {
4394                 ret = -EINVAL;
4395                 goto out;
4396         }
4397
4398         event_state = &event_mgr->event_state[type];
4399         event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno);
4400         atomic64_inc(&event_state->count);
4401
4402 out:
4403         if (ret && caller)
4404                 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n",
4405                          (int)type, caller, ret);
4406
4407         return ret;
4408 }
4409
4410 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type)
4411 {
4412         struct ras_event_manager *event_mgr;
4413         u64 id;
4414
4415         if (type >= RAS_EVENT_TYPE_COUNT)
4416                 return RAS_EVENT_INVALID_ID;
4417
4418         switch (type) {
4419         case RAS_EVENT_TYPE_FATAL:
4420         case RAS_EVENT_TYPE_POISON_CREATION:
4421         case RAS_EVENT_TYPE_POISON_CONSUMPTION:
4422                 event_mgr = __get_ras_event_mgr(adev);
4423                 if (!event_mgr)
4424                         return RAS_EVENT_INVALID_ID;
4425
4426                 id = event_mgr->event_state[type].last_seqno;
4427                 break;
4428         case RAS_EVENT_TYPE_INVALID:
4429         default:
4430                 id = RAS_EVENT_INVALID_ID;
4431                 break;
4432         }
4433
4434         return id;
4435 }
4436
4437 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
4438 {
4439         if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
4440                 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4441                 enum ras_event_type type = RAS_EVENT_TYPE_FATAL;
4442                 u64 event_id;
4443
4444                 if (amdgpu_ras_mark_ras_event(adev, type))
4445                         return;
4446
4447                 event_id = amdgpu_ras_acquire_event_id(adev, type);
4448
4449                 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error"
4450                               "(ERREVENT_ATHUB_INTERRUPT) detected!\n");
4451
4452                 amdgpu_ras_set_fed(adev, true);
4453                 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4454                 amdgpu_ras_reset_gpu(adev);
4455         }
4456 }
4457
4458 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
4459 {
4460         if (adev->asic_type == CHIP_VEGA20 &&
4461             adev->pm.fw_version <= 0x283400) {
4462                 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
4463                                 amdgpu_ras_intr_triggered();
4464         }
4465
4466         return false;
4467 }
4468
4469 void amdgpu_release_ras_context(struct amdgpu_device *adev)
4470 {
4471         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4472
4473         if (!con)
4474                 return;
4475
4476         if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
4477                 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
4478                 amdgpu_ras_set_context(adev, NULL);
4479                 kfree(con);
4480         }
4481 }
4482
4483 #ifdef CONFIG_X86_MCE_AMD
4484 static struct amdgpu_device *find_adev(uint32_t node_id)
4485 {
4486         int i;
4487         struct amdgpu_device *adev = NULL;
4488
4489         for (i = 0; i < mce_adev_list.num_gpu; i++) {
4490                 adev = mce_adev_list.devs[i];
4491
4492                 if (adev && adev->gmc.xgmi.connected_to_cpu &&
4493                     adev->gmc.xgmi.physical_node_id == node_id)
4494                         break;
4495                 adev = NULL;
4496         }
4497
4498         return adev;
4499 }
4500
4501 #define GET_MCA_IPID_GPUID(m)   (((m) >> 44) & 0xF)
4502 #define GET_UMC_INST(m)         (((m) >> 21) & 0x7)
4503 #define GET_CHAN_INDEX(m)       ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
4504 #define GPU_ID_OFFSET           8
4505
4506 static int amdgpu_bad_page_notifier(struct notifier_block *nb,
4507                                     unsigned long val, void *data)
4508 {
4509         struct mce *m = (struct mce *)data;
4510         struct amdgpu_device *adev = NULL;
4511         uint32_t gpu_id = 0;
4512         uint32_t umc_inst = 0, ch_inst = 0;
4513
4514         /*
4515          * If the error was generated in UMC_V2, which belongs to GPU UMCs,
4516          * and error occurred in DramECC (Extended error code = 0) then only
4517          * process the error, else bail out.
4518          */
4519         if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
4520                     (XEC(m->status, 0x3f) == 0x0)))
4521                 return NOTIFY_DONE;
4522
4523         /*
4524          * If it is correctable error, return.
4525          */
4526         if (mce_is_correctable(m))
4527                 return NOTIFY_OK;
4528
4529         /*
4530          * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
4531          */
4532         gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
4533
4534         adev = find_adev(gpu_id);
4535         if (!adev) {
4536                 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
4537                                                                 gpu_id);
4538                 return NOTIFY_DONE;
4539         }
4540
4541         /*
4542          * If it is uncorrectable error, then find out UMC instance and
4543          * channel index.
4544          */
4545         umc_inst = GET_UMC_INST(m->ipid);
4546         ch_inst = GET_CHAN_INDEX(m->ipid);
4547
4548         dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
4549                              umc_inst, ch_inst);
4550
4551         if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst))
4552                 return NOTIFY_OK;
4553         else
4554                 return NOTIFY_DONE;
4555 }
4556
4557 static struct notifier_block amdgpu_bad_page_nb = {
4558         .notifier_call  = amdgpu_bad_page_notifier,
4559         .priority       = MCE_PRIO_UC,
4560 };
4561
4562 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
4563 {
4564         /*
4565          * Add the adev to the mce_adev_list.
4566          * During mode2 reset, amdgpu device is temporarily
4567          * removed from the mgpu_info list which can cause
4568          * page retirement to fail.
4569          * Use this list instead of mgpu_info to find the amdgpu
4570          * device on which the UMC error was reported.
4571          */
4572         mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
4573
4574         /*
4575          * Register the x86 notifier only once
4576          * with MCE subsystem.
4577          */
4578         if (notifier_registered == false) {
4579                 mce_register_decode_chain(&amdgpu_bad_page_nb);
4580                 notifier_registered = true;
4581         }
4582 }
4583 #endif
4584
4585 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
4586 {
4587         if (!adev)
4588                 return NULL;
4589
4590         return adev->psp.ras_context.ras;
4591 }
4592
4593 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
4594 {
4595         if (!adev)
4596                 return -EINVAL;
4597
4598         adev->psp.ras_context.ras = ras_con;
4599         return 0;
4600 }
4601
4602 /* check if ras is supported on block, say, sdma, gfx */
4603 int amdgpu_ras_is_supported(struct amdgpu_device *adev,
4604                 unsigned int block)
4605 {
4606         int ret = 0;
4607         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4608
4609         if (block >= AMDGPU_RAS_BLOCK_COUNT)
4610                 return 0;
4611
4612         ret = ras && (adev->ras_enabled & (1 << block));
4613
4614         /* For the special asic with mem ecc enabled but sram ecc
4615          * not enabled, even if the ras block is not supported on
4616          * .ras_enabled, if the asic supports poison mode and the
4617          * ras block has ras configuration, it can be considered
4618          * that the ras block supports ras function.
4619          */
4620         if (!ret &&
4621             (block == AMDGPU_RAS_BLOCK__GFX ||
4622              block == AMDGPU_RAS_BLOCK__SDMA ||
4623              block == AMDGPU_RAS_BLOCK__VCN ||
4624              block == AMDGPU_RAS_BLOCK__JPEG) &&
4625                 (amdgpu_ras_mask & (1 << block)) &&
4626             amdgpu_ras_is_poison_mode_supported(adev) &&
4627             amdgpu_ras_get_ras_block(adev, block, 0))
4628                 ret = 1;
4629
4630         return ret;
4631 }
4632
4633 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
4634 {
4635         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4636
4637         /* mode1 is the only selection for RMA status */
4638         if (amdgpu_ras_is_rma(adev)) {
4639                 ras->gpu_reset_flags = 0;
4640                 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4641         }
4642
4643         if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) {
4644                 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
4645                 int hive_ras_recovery = 0;
4646
4647                 if (hive) {
4648                         hive_ras_recovery = atomic_read(&hive->ras_recovery);
4649                         amdgpu_put_xgmi_hive(hive);
4650                 }
4651                 /* In the case of multiple GPUs, after a GPU has started
4652                  * resetting all GPUs on hive, other GPUs do not need to
4653                  * trigger GPU reset again.
4654                  */
4655                 if (!hive_ras_recovery)
4656                         amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4657                 else
4658                         atomic_set(&ras->in_recovery, 0);
4659         } else {
4660                 flush_work(&ras->recovery_work);
4661                 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4662         }
4663
4664         return 0;
4665 }
4666
4667 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable)
4668 {
4669         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4670         int ret = 0;
4671
4672         if (con) {
4673                 ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4674                 if (!ret)
4675                         con->is_aca_debug_mode = enable;
4676         }
4677
4678         return ret;
4679 }
4680
4681 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable)
4682 {
4683         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4684         int ret = 0;
4685
4686         if (con) {
4687                 if (amdgpu_aca_is_enabled(adev))
4688                         ret = amdgpu_aca_smu_set_debug_mode(adev, enable);
4689                 else
4690                         ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4691                 if (!ret)
4692                         con->is_aca_debug_mode = enable;
4693         }
4694
4695         return ret;
4696 }
4697
4698 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev)
4699 {
4700         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4701         const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
4702         const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4703
4704         if (!con)
4705                 return false;
4706
4707         if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) ||
4708             (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode))
4709                 return con->is_aca_debug_mode;
4710         else
4711                 return true;
4712 }
4713
4714 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev,
4715                                      unsigned int *error_query_mode)
4716 {
4717         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4718         const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4719         const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
4720
4721         if (!con) {
4722                 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY;
4723                 return false;
4724         }
4725
4726         if (amdgpu_sriov_vf(adev)) {
4727                 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY;
4728         } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) {
4729                 *error_query_mode =
4730                         (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY;
4731         } else {
4732                 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY;
4733         }
4734
4735         return true;
4736 }
4737
4738 /* Register each ip ras block into amdgpu ras */
4739 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
4740                 struct amdgpu_ras_block_object *ras_block_obj)
4741 {
4742         struct amdgpu_ras_block_list *ras_node;
4743         if (!adev || !ras_block_obj)
4744                 return -EINVAL;
4745
4746         ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
4747         if (!ras_node)
4748                 return -ENOMEM;
4749
4750         INIT_LIST_HEAD(&ras_node->node);
4751         ras_node->ras_obj = ras_block_obj;
4752         list_add_tail(&ras_node->node, &adev->ras_list);
4753
4754         return 0;
4755 }
4756
4757 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name)
4758 {
4759         if (!err_type_name)
4760                 return;
4761
4762         switch (err_type) {
4763         case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE:
4764                 sprintf(err_type_name, "correctable");
4765                 break;
4766         case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE:
4767                 sprintf(err_type_name, "uncorrectable");
4768                 break;
4769         default:
4770                 sprintf(err_type_name, "unknown");
4771                 break;
4772         }
4773 }
4774
4775 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev,
4776                                          const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4777                                          uint32_t instance,
4778                                          uint32_t *memory_id)
4779 {
4780         uint32_t err_status_lo_data, err_status_lo_offset;
4781
4782         if (!reg_entry)
4783                 return false;
4784
4785         err_status_lo_offset =
4786                 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4787                                             reg_entry->seg_lo, reg_entry->reg_lo);
4788         err_status_lo_data = RREG32(err_status_lo_offset);
4789
4790         if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) &&
4791             !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG))
4792                 return false;
4793
4794         *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID);
4795
4796         return true;
4797 }
4798
4799 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev,
4800                                        const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4801                                        uint32_t instance,
4802                                        unsigned long *err_cnt)
4803 {
4804         uint32_t err_status_hi_data, err_status_hi_offset;
4805
4806         if (!reg_entry)
4807                 return false;
4808
4809         err_status_hi_offset =
4810                 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4811                                             reg_entry->seg_hi, reg_entry->reg_hi);
4812         err_status_hi_data = RREG32(err_status_hi_offset);
4813
4814         if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) &&
4815             !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG))
4816                 /* keep the check here in case we need to refer to the result later */
4817                 dev_dbg(adev->dev, "Invalid err_info field\n");
4818
4819         /* read err count */
4820         *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT);
4821
4822         return true;
4823 }
4824
4825 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev,
4826                                            const struct amdgpu_ras_err_status_reg_entry *reg_list,
4827                                            uint32_t reg_list_size,
4828                                            const struct amdgpu_ras_memory_id_entry *mem_list,
4829                                            uint32_t mem_list_size,
4830                                            uint32_t instance,
4831                                            uint32_t err_type,
4832                                            unsigned long *err_count)
4833 {
4834         uint32_t memory_id;
4835         unsigned long err_cnt;
4836         char err_type_name[16];
4837         uint32_t i, j;
4838
4839         for (i = 0; i < reg_list_size; i++) {
4840                 /* query memory_id from err_status_lo */
4841                 if (!amdgpu_ras_inst_get_memory_id_field(adev, &reg_list[i],
4842                                                          instance, &memory_id))
4843                         continue;
4844
4845                 /* query err_cnt from err_status_hi */
4846                 if (!amdgpu_ras_inst_get_err_cnt_field(adev, &reg_list[i],
4847                                                        instance, &err_cnt) ||
4848                     !err_cnt)
4849                         continue;
4850
4851                 *err_count += err_cnt;
4852
4853                 /* log the errors */
4854                 amdgpu_ras_get_error_type_name(err_type, err_type_name);
4855                 if (!mem_list) {
4856                         /* memory_list is not supported */
4857                         dev_info(adev->dev,
4858                                  "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n",
4859                                  err_cnt, err_type_name,
4860                                  reg_list[i].block_name,
4861                                  instance, memory_id);
4862                 } else {
4863                         for (j = 0; j < mem_list_size; j++) {
4864                                 if (memory_id == mem_list[j].memory_id) {
4865                                         dev_info(adev->dev,
4866                                                  "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n",
4867                                                  err_cnt, err_type_name,
4868                                                  reg_list[i].block_name,
4869                                                  instance, mem_list[j].name);
4870                                         break;
4871                                 }
4872                         }
4873                 }
4874         }
4875 }
4876
4877 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev,
4878                                            const struct amdgpu_ras_err_status_reg_entry *reg_list,
4879                                            uint32_t reg_list_size,
4880                                            uint32_t instance)
4881 {
4882         uint32_t err_status_lo_offset, err_status_hi_offset;
4883         uint32_t i;
4884
4885         for (i = 0; i < reg_list_size; i++) {
4886                 err_status_lo_offset =
4887                         AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4888                                                     reg_list[i].seg_lo, reg_list[i].reg_lo);
4889                 err_status_hi_offset =
4890                         AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4891                                                     reg_list[i].seg_hi, reg_list[i].reg_hi);
4892                 WREG32(err_status_lo_offset, 0);
4893                 WREG32(err_status_hi_offset, 0);
4894         }
4895 }
4896
4897 int amdgpu_ras_error_data_init(struct ras_err_data *err_data)
4898 {
4899         memset(err_data, 0, sizeof(*err_data));
4900
4901         INIT_LIST_HEAD(&err_data->err_node_list);
4902
4903         return 0;
4904 }
4905
4906 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node)
4907 {
4908         if (!err_node)
4909                 return;
4910
4911         list_del(&err_node->node);
4912         kvfree(err_node);
4913 }
4914
4915 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data)
4916 {
4917         struct ras_err_node *err_node, *tmp;
4918
4919         list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node)
4920                 amdgpu_ras_error_node_release(err_node);
4921 }
4922
4923 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data,
4924                                                              struct amdgpu_smuio_mcm_config_info *mcm_info)
4925 {
4926         struct ras_err_node *err_node;
4927         struct amdgpu_smuio_mcm_config_info *ref_id;
4928
4929         if (!err_data || !mcm_info)
4930                 return NULL;
4931
4932         for_each_ras_error(err_node, err_data) {
4933                 ref_id = &err_node->err_info.mcm_info;
4934
4935                 if (mcm_info->socket_id == ref_id->socket_id &&
4936                     mcm_info->die_id == ref_id->die_id)
4937                         return err_node;
4938         }
4939
4940         return NULL;
4941 }
4942
4943 static struct ras_err_node *amdgpu_ras_error_node_new(void)
4944 {
4945         struct ras_err_node *err_node;
4946
4947         err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL);
4948         if (!err_node)
4949                 return NULL;
4950
4951         INIT_LIST_HEAD(&err_node->node);
4952
4953         return err_node;
4954 }
4955
4956 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b)
4957 {
4958         struct ras_err_node *nodea = container_of(a, struct ras_err_node, node);
4959         struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node);
4960         struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info;
4961         struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info;
4962
4963         if (unlikely(infoa->socket_id != infob->socket_id))
4964                 return infoa->socket_id - infob->socket_id;
4965         else
4966                 return infoa->die_id - infob->die_id;
4967
4968         return 0;
4969 }
4970
4971 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data,
4972                                 struct amdgpu_smuio_mcm_config_info *mcm_info)
4973 {
4974         struct ras_err_node *err_node;
4975
4976         err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info);
4977         if (err_node)
4978                 return &err_node->err_info;
4979
4980         err_node = amdgpu_ras_error_node_new();
4981         if (!err_node)
4982                 return NULL;
4983
4984         memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info));
4985
4986         err_data->err_list_count++;
4987         list_add_tail(&err_node->node, &err_data->err_node_list);
4988         list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp);
4989
4990         return &err_node->err_info;
4991 }
4992
4993 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data,
4994                                         struct amdgpu_smuio_mcm_config_info *mcm_info,
4995                                         u64 count)
4996 {
4997         struct ras_err_info *err_info;
4998
4999         if (!err_data || !mcm_info)
5000                 return -EINVAL;
5001
5002         if (!count)
5003                 return 0;
5004
5005         err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5006         if (!err_info)
5007                 return -EINVAL;
5008
5009         err_info->ue_count += count;
5010         err_data->ue_count += count;
5011
5012         return 0;
5013 }
5014
5015 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data,
5016                                         struct amdgpu_smuio_mcm_config_info *mcm_info,
5017                                         u64 count)
5018 {
5019         struct ras_err_info *err_info;
5020
5021         if (!err_data || !mcm_info)
5022                 return -EINVAL;
5023
5024         if (!count)
5025                 return 0;
5026
5027         err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5028         if (!err_info)
5029                 return -EINVAL;
5030
5031         err_info->ce_count += count;
5032         err_data->ce_count += count;
5033
5034         return 0;
5035 }
5036
5037 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data,
5038                                         struct amdgpu_smuio_mcm_config_info *mcm_info,
5039                                         u64 count)
5040 {
5041         struct ras_err_info *err_info;
5042
5043         if (!err_data || !mcm_info)
5044                 return -EINVAL;
5045
5046         if (!count)
5047                 return 0;
5048
5049         err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5050         if (!err_info)
5051                 return -EINVAL;
5052
5053         err_info->de_count += count;
5054         err_data->de_count += count;
5055
5056         return 0;
5057 }
5058
5059 #define mmMP0_SMN_C2PMSG_92     0x1609C
5060 #define mmMP0_SMN_C2PMSG_126    0x160BE
5061 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev,
5062                                                  u32 instance)
5063 {
5064         u32 socket_id, aid_id, hbm_id;
5065         u32 fw_status;
5066         u32 boot_error;
5067         u64 reg_addr;
5068
5069         /* The pattern for smn addressing in other SOC could be different from
5070          * the one for aqua_vanjaram. We should revisit the code if the pattern
5071          * is changed. In such case, replace the aqua_vanjaram implementation
5072          * with more common helper */
5073         reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5074                    aqua_vanjaram_encode_ext_smn_addressing(instance);
5075         fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5076
5077         reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) +
5078                    aqua_vanjaram_encode_ext_smn_addressing(instance);
5079         boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5080
5081         socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error);
5082         aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error);
5083         hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1);
5084
5085         if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error))
5086                 dev_info(adev->dev,
5087                          "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n",
5088                          socket_id, aid_id, hbm_id, fw_status);
5089
5090         if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error))
5091                 dev_info(adev->dev,
5092                          "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n",
5093                          socket_id, aid_id, fw_status);
5094
5095         if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error))
5096                 dev_info(adev->dev,
5097                          "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n",
5098                          socket_id, aid_id, fw_status);
5099
5100         if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error))
5101                 dev_info(adev->dev,
5102                          "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n",
5103                          socket_id, aid_id, fw_status);
5104
5105         if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error))
5106                 dev_info(adev->dev,
5107                          "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n",
5108                          socket_id, aid_id, fw_status);
5109
5110         if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error))
5111                 dev_info(adev->dev,
5112                          "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n",
5113                          socket_id, aid_id, fw_status);
5114
5115         if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error))
5116                 dev_info(adev->dev,
5117                          "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n",
5118                          socket_id, aid_id, hbm_id, fw_status);
5119
5120         if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error))
5121                 dev_info(adev->dev,
5122                          "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n",
5123                          socket_id, aid_id, hbm_id, fw_status);
5124
5125         if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error))
5126                 dev_info(adev->dev,
5127                          "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n",
5128                          socket_id, aid_id, fw_status);
5129
5130         if (AMDGPU_RAS_GPU_ERR_UNKNOWN(boot_error))
5131                 dev_info(adev->dev,
5132                          "socket: %d, aid: %d, fw_status: 0x%x, unknown boot time errors\n",
5133                          socket_id, aid_id, fw_status);
5134 }
5135
5136 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev,
5137                                            u32 instance)
5138 {
5139         u64 reg_addr;
5140         u32 reg_data;
5141         int retry_loop;
5142
5143         reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5144                    aqua_vanjaram_encode_ext_smn_addressing(instance);
5145
5146         for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) {
5147                 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5148                 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS)
5149                         return false;
5150                 else
5151                         msleep(1);
5152         }
5153
5154         return true;
5155 }
5156
5157 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances)
5158 {
5159         u32 i;
5160
5161         for (i = 0; i < num_instances; i++) {
5162                 if (amdgpu_ras_boot_error_detected(adev, i))
5163                         amdgpu_ras_boot_time_error_reporting(adev, i);
5164         }
5165 }
5166
5167 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn)
5168 {
5169         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5170         struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
5171         uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT;
5172         int ret = 0;
5173
5174         mutex_lock(&con->page_rsv_lock);
5175         ret = amdgpu_vram_mgr_query_page_status(mgr, start);
5176         if (ret == -ENOENT)
5177                 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE);
5178         mutex_unlock(&con->page_rsv_lock);
5179
5180         return ret;
5181 }
5182
5183 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id,
5184                                 const char *fmt, ...)
5185 {
5186         struct va_format vaf;
5187         va_list args;
5188
5189         va_start(args, fmt);
5190         vaf.fmt = fmt;
5191         vaf.va = &args;
5192
5193         if (RAS_EVENT_ID_IS_VALID(event_id))
5194                 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf);
5195         else
5196                 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf);
5197
5198         va_end(args);
5199 }
5200
5201 bool amdgpu_ras_is_rma(struct amdgpu_device *adev)
5202 {
5203         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5204
5205         if (!con)
5206                 return false;
5207
5208         return con->is_rma;
5209 }