drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24 #include <linux/debugfs.h>
  25 #include <linux/list.h>
  26 #include <linux/module.h>
  27 #include <linux/uaccess.h>
  28 #include <linux/reboot.h>
  29 #include <linux/syscalls.h>
  30 #include <linux/pm_runtime.h>
  31
  32 #include "amdgpu.h"
  33 #include "amdgpu_ras.h"
  34 #include "amdgpu_atomfirmware.h"
  35 #include "amdgpu_xgmi.h"
  36 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
  37 #include "atom.h"
  38 #include "amdgpu_reset.h"
  39
  40 #ifdef CONFIG_X86_MCE_AMD
  41 #include <asm/mce.h>
  42
  43 static bool notifier_registered;
  44 #endif
  45 static const char *RAS_FS_NAME = "ras";
  46
  47 const char *ras_error_string[] = {
  48         "none",
  49         "parity",
  50         "single_correctable",
  51         "multi_uncorrectable",
  52         "poison",
  53 };
  54
  55 const char *ras_block_string[] = {
  56         "umc",
  57         "sdma",
  58         "gfx",
  59         "mmhub",
  60         "athub",
  61         "pcie_bif",
  62         "hdp",
  63         "xgmi_wafl",
  64         "df",
  65         "smn",
  66         "sem",
  67         "mp0",
  68         "mp1",
  69         "fuse",
  70         "mca",
  71         "vcn",
  72         "jpeg",
  73 };
  74
  75 const char *ras_mca_block_string[] = {
  76         "mca_mp0",
  77         "mca_mp1",
  78         "mca_mpio",
  79         "mca_iohc",
  80 };
  81
  82 struct amdgpu_ras_block_list {
  83         /* ras block link */
  84         struct list_head node;
  85
  86         struct amdgpu_ras_block_object *ras_obj;
  87 };
  88
  89 const char *get_ras_block_str(struct ras_common_if *ras_block)
  90 {
  91         if (!ras_block)
  92                 return "NULL";
  93
  94         if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT)
  95                 return "OUT OF RANGE";
  96
  97         if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
  98                 return ras_mca_block_string[ras_block->sub_block_index];
  99
 100         return ras_block_string[ras_block->block];
 101 }
 102
 103 #define ras_block_str(_BLOCK_) \
 104         (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
 105
 106 #define ras_err_str(i) (ras_error_string[ffs(i)])
 107
 108 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
 109
 110 /* inject address is 52 bits */
 111 #define RAS_UMC_INJECT_ADDR_LIMIT       (0x1ULL << 52)
 112
 113 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */
 114 #define RAS_BAD_PAGE_COVER              (100 * 1024 * 1024ULL)
 115
 116 enum amdgpu_ras_retire_page_reservation {
 117         AMDGPU_RAS_RETIRE_PAGE_RESERVED,
 118         AMDGPU_RAS_RETIRE_PAGE_PENDING,
 119         AMDGPU_RAS_RETIRE_PAGE_FAULT,
 120 };
 121
 122 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
 123
 124 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
 125                                 uint64_t addr);
 126 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
 127                                 uint64_t addr);
 128 #ifdef CONFIG_X86_MCE_AMD
 129 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
 130 struct mce_notifier_adev_list {
 131         struct amdgpu_device *devs[MAX_GPU_INSTANCE];
 132         int num_gpu;
 133 };
 134 static struct mce_notifier_adev_list mce_adev_list;
 135 #endif
 136
 137 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
 138 {
 139         if (adev && amdgpu_ras_get_context(adev))
 140                 amdgpu_ras_get_context(adev)->error_query_ready = ready;
 141 }
 142
 143 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
 144 {
 145         if (adev && amdgpu_ras_get_context(adev))
 146                 return amdgpu_ras_get_context(adev)->error_query_ready;
 147
 148         return false;
 149 }
 150
 151 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
 152 {
 153         struct ras_err_data err_data = {0, 0, 0, NULL};
 154         struct eeprom_table_record err_rec;
 155
 156         if ((address >= adev->gmc.mc_vram_size) ||
 157             (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
 158                 dev_warn(adev->dev,
 159                          "RAS WARN: input address 0x%llx is invalid.\n",
 160                          address);
 161                 return -EINVAL;
 162         }
 163
 164         if (amdgpu_ras_check_bad_page(adev, address)) {
 165                 dev_warn(adev->dev,
 166                          "RAS WARN: 0x%llx has already been marked as bad page!\n",
 167                          address);
 168                 return 0;
 169         }
 170
 171         memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
 172         err_data.err_addr = &err_rec;
 173         amdgpu_umc_fill_error_record(&err_data, address,
 174                         (address >> AMDGPU_GPU_PAGE_SHIFT), 0, 0);
 175
 176         if (amdgpu_bad_page_threshold != 0) {
 177                 amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
 178                                          err_data.err_addr_cnt);
 179                 amdgpu_ras_save_bad_pages(adev);
 180         }
 181
 182         dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
 183         dev_warn(adev->dev, "Clear EEPROM:\n");
 184         dev_warn(adev->dev, "    echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
 185
 186         return 0;
 187 }
 188
 189 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
 190                                         size_t size, loff_t *pos)
 191 {
 192         struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
 193         struct ras_query_if info = {
 194                 .head = obj->head,
 195         };
 196         ssize_t s;
 197         char val[128];
 198
 199         if (amdgpu_ras_query_error_status(obj->adev, &info))
 200                 return -EINVAL;
 201
 202         /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
 203         if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
 204             obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
 205                 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
 206                         dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
 207         }
 208
 209         s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
 210                         "ue", info.ue_count,
 211                         "ce", info.ce_count);
 212         if (*pos >= s)
 213                 return 0;
 214
 215         s -= *pos;
 216         s = min_t(u64, s, size);
 217
 218
 219         if (copy_to_user(buf, &val[*pos], s))
 220                 return -EINVAL;
 221
 222         *pos += s;
 223
 224         return s;
 225 }
 226
 227 static const struct file_operations amdgpu_ras_debugfs_ops = {
 228         .owner = THIS_MODULE,
 229         .read = amdgpu_ras_debugfs_read,
 230         .write = NULL,
 231         .llseek = default_llseek
 232 };
 233
 234 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
 235 {
 236         int i;
 237
 238         for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
 239                 *block_id = i;
 240                 if (strcmp(name, ras_block_string[i]) == 0)
 241                         return 0;
 242         }
 243         return -EINVAL;
 244 }
 245
 246 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
 247                 const char __user *buf, size_t size,
 248                 loff_t *pos, struct ras_debug_if *data)
 249 {
 250         ssize_t s = min_t(u64, 64, size);
 251         char str[65];
 252         char block_name[33];
 253         char err[9] = "ue";
 254         int op = -1;
 255         int block_id;
 256         uint32_t sub_block;
 257         u64 address, value;
 258
 259         if (*pos)
 260                 return -EINVAL;
 261         *pos = size;
 262
 263         memset(str, 0, sizeof(str));
 264         memset(data, 0, sizeof(*data));
 265
 266         if (copy_from_user(str, buf, s))
 267                 return -EINVAL;
 268
 269         if (sscanf(str, "disable %32s", block_name) == 1)
 270                 op = 0;
 271         else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
 272                 op = 1;
 273         else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
 274                 op = 2;
 275         else if (strstr(str, "retire_page") != NULL)
 276                 op = 3;
 277         else if (str[0] && str[1] && str[2] && str[3])
 278                 /* ascii string, but commands are not matched. */
 279                 return -EINVAL;
 280
 281         if (op != -1) {
 282                 if (op == 3) {
 283                         if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
 284                             sscanf(str, "%*s %llu", &address) != 1)
 285                                 return -EINVAL;
 286
 287                         data->op = op;
 288                         data->inject.address = address;
 289
 290                         return 0;
 291                 }
 292
 293                 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
 294                         return -EINVAL;
 295
 296                 data->head.block = block_id;
 297                 /* only ue and ce errors are supported */
 298                 if (!memcmp("ue", err, 2))
 299                         data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
 300                 else if (!memcmp("ce", err, 2))
 301                         data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
 302                 else
 303                         return -EINVAL;
 304
 305                 data->op = op;
 306
 307                 if (op == 2) {
 308                         if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
 309                                    &sub_block, &address, &value) != 3 &&
 310                             sscanf(str, "%*s %*s %*s %u %llu %llu",
 311                                    &sub_block, &address, &value) != 3)
 312                                 return -EINVAL;
 313                         data->head.sub_block_index = sub_block;
 314                         data->inject.address = address;
 315                         data->inject.value = value;
 316                 }
 317         } else {
 318                 if (size < sizeof(*data))
 319                         return -EINVAL;
 320
 321                 if (copy_from_user(data, buf, sizeof(*data)))
 322                         return -EINVAL;
 323         }
 324
 325         return 0;
 326 }
 327
 328 /**
 329  * DOC: AMDGPU RAS debugfs control interface
 330  *
 331  * The control interface accepts struct ras_debug_if which has two members.
 332  *
 333  * First member: ras_debug_if::head or ras_debug_if::inject.
 334  *
 335  * head is used to indicate which IP block will be under control.
 336  *
 337  * head has four members, they are block, type, sub_block_index, name.
 338  * block: which IP will be under control.
 339  * type: what kind of error will be enabled/disabled/injected.
 340  * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
 341  * name: the name of IP.
 342  *
 343  * inject has two more members than head, they are address, value.
 344  * As their names indicate, inject operation will write the
 345  * value to the address.
 346  *
 347  * The second member: struct ras_debug_if::op.
 348  * It has three kinds of operations.
 349  *
 350  * - 0: disable RAS on the block. Take ::head as its data.
 351  * - 1: enable RAS on the block. Take ::head as its data.
 352  * - 2: inject errors on the block. Take ::inject as its data.
 353  *
 354  * How to use the interface?
 355  *
 356  * In a program
 357  *
 358  * Copy the struct ras_debug_if in your code and initialize it.
 359  * Write the struct to the control interface.
 360  *
 361  * From shell
 362  *
 363  * .. code-block:: bash
 364  *
 365  *      echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
 366  *      echo "enable  <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
 367  *      echo "inject  <block> <error> <sub-block> <address> <value> > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
 368  *
 369  * Where N, is the card which you want to affect.
 370  *
 371  * "disable" requires only the block.
 372  * "enable" requires the block and error type.
 373  * "inject" requires the block, error type, address, and value.
 374  *
 375  * The block is one of: umc, sdma, gfx, etc.
 376  *      see ras_block_string[] for details
 377  *
 378  * The error type is one of: ue, ce, where,
 379  *      ue is multi-uncorrectable
 380  *      ce is single-correctable
 381  *
 382  * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
 383  * The address and value are hexadecimal numbers, leading 0x is optional.
 384  *
 385  * For instance,
 386  *
 387  * .. code-block:: bash
 388  *
 389  *      echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
 390  *      echo inject umc ce 0 0 0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
 391  *      echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
 392  *
 393  * How to check the result of the operation?
 394  *
 395  * To check disable/enable, see "ras" features at,
 396  * /sys/class/drm/card[0/1/2...]/device/ras/features
 397  *
 398  * To check inject, see the corresponding error count at,
 399  * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
 400  *
 401  * .. note::
 402  *      Operations are only allowed on blocks which are supported.
 403  *      Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
 404  *      to see which blocks support RAS on a particular asic.
 405  *
 406  */
 407 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
 408                                              const char __user *buf,
 409                                              size_t size, loff_t *pos)
 410 {
 411         struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
 412         struct ras_debug_if data;
 413         int ret = 0;
 414
 415         if (!amdgpu_ras_get_error_query_ready(adev)) {
 416                 dev_warn(adev->dev, "RAS WARN: error injection "
 417                                 "currently inaccessible\n");
 418                 return size;
 419         }
 420
 421         ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
 422         if (ret)
 423                 return ret;
 424
 425         if (data.op == 3) {
 426                 ret = amdgpu_reserve_page_direct(adev, data.inject.address);
 427                 if (!ret)
 428                         return size;
 429                 else
 430                         return ret;
 431         }
 432
 433         if (!amdgpu_ras_is_supported(adev, data.head.block))
 434                 return -EINVAL;
 435
 436         switch (data.op) {
 437         case 0:
 438                 ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
 439                 break;
 440         case 1:
 441                 ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
 442                 break;
 443         case 2:
 444                 if ((data.inject.address >= adev->gmc.mc_vram_size) ||
 445                     (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
 446                         dev_warn(adev->dev, "RAS WARN: input address "
 447                                         "0x%llx is invalid.",
 448                                         data.inject.address);
 449                         ret = -EINVAL;
 450                         break;
 451                 }
 452
 453                 /* umc ce/ue error injection for a bad page is not allowed */
 454                 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
 455                     amdgpu_ras_check_bad_page(adev, data.inject.address)) {
 456                         dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
 457                                  "already been marked as bad!\n",
 458                                  data.inject.address);
 459                         break;
 460                 }
 461
 462                 /* data.inject.address is offset instead of absolute gpu address */
 463                 ret = amdgpu_ras_error_inject(adev, &data.inject);
 464                 break;
 465         default:
 466                 ret = -EINVAL;
 467                 break;
 468         }
 469
 470         if (ret)
 471                 return ret;
 472
 473         return size;
 474 }
 475
 476 /**
 477  * DOC: AMDGPU RAS debugfs EEPROM table reset interface
 478  *
 479  * Some boards contain an EEPROM which is used to persistently store a list of
 480  * bad pages which experiences ECC errors in vram.  This interface provides
 481  * a way to reset the EEPROM, e.g., after testing error injection.
 482  *
 483  * Usage:
 484  *
 485  * .. code-block:: bash
 486  *
 487  *      echo 1 > ../ras/ras_eeprom_reset
 488  *
 489  * will reset EEPROM table to 0 entries.
 490  *
 491  */
 492 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
 493                                                const char __user *buf,
 494                                                size_t size, loff_t *pos)
 495 {
 496         struct amdgpu_device *adev =
 497                 (struct amdgpu_device *)file_inode(f)->i_private;
 498         int ret;
 499
 500         ret = amdgpu_ras_eeprom_reset_table(
 501                 &(amdgpu_ras_get_context(adev)->eeprom_control));
 502
 503         if (!ret) {
 504                 /* Something was written to EEPROM.
 505                  */
 506                 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
 507                 return size;
 508         } else {
 509                 return ret;
 510         }
 511 }
 512
 513 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
 514         .owner = THIS_MODULE,
 515         .read = NULL,
 516         .write = amdgpu_ras_debugfs_ctrl_write,
 517         .llseek = default_llseek
 518 };
 519
 520 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
 521         .owner = THIS_MODULE,
 522         .read = NULL,
 523         .write = amdgpu_ras_debugfs_eeprom_write,
 524         .llseek = default_llseek
 525 };
 526
 527 /**
 528  * DOC: AMDGPU RAS sysfs Error Count Interface
 529  *
 530  * It allows the user to read the error count for each IP block on the gpu through
 531  * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
 532  *
 533  * It outputs the multiple lines which report the uncorrected (ue) and corrected
 534  * (ce) error counts.
 535  *
 536  * The format of one line is below,
 537  *
 538  * [ce|ue]: count
 539  *
 540  * Example:
 541  *
 542  * .. code-block:: bash
 543  *
 544  *      ue: 0
 545  *      ce: 1
 546  *
 547  */
 548 static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
 549                 struct device_attribute *attr, char *buf)
 550 {
 551         struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
 552         struct ras_query_if info = {
 553                 .head = obj->head,
 554         };
 555
 556         if (!amdgpu_ras_get_error_query_ready(obj->adev))
 557                 return sysfs_emit(buf, "Query currently inaccessible\n");
 558
 559         if (amdgpu_ras_query_error_status(obj->adev, &info))
 560                 return -EINVAL;
 561
 562         if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
 563             obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
 564                 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
 565                         dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
 566         }
 567
 568         return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
 569                           "ce", info.ce_count);
 570 }
 571
 572 /* obj begin */
 573
 574 #define get_obj(obj) do { (obj)->use++; } while (0)
 575 #define alive_obj(obj) ((obj)->use)
 576
 577 static inline void put_obj(struct ras_manager *obj)
 578 {
 579         if (obj && (--obj->use == 0))
 580                 list_del(&obj->node);
 581         if (obj && (obj->use < 0))
 582                 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
 583 }
 584
 585 /* make one obj and return it. */
 586 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
 587                 struct ras_common_if *head)
 588 {
 589         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 590         struct ras_manager *obj;
 591
 592         if (!adev->ras_enabled || !con)
 593                 return NULL;
 594
 595         if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
 596                 return NULL;
 597
 598         if (head->block == AMDGPU_RAS_BLOCK__MCA) {
 599                 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
 600                         return NULL;
 601
 602                 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
 603         } else
 604                 obj = &con->objs[head->block];
 605
 606         /* already exist. return obj? */
 607         if (alive_obj(obj))
 608                 return NULL;
 609
 610         obj->head = *head;
 611         obj->adev = adev;
 612         list_add(&obj->node, &con->head);
 613         get_obj(obj);
 614
 615         return obj;
 616 }
 617
 618 /* return an obj equal to head, or the first when head is NULL */
 619 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
 620                 struct ras_common_if *head)
 621 {
 622         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 623         struct ras_manager *obj;
 624         int i;
 625
 626         if (!adev->ras_enabled || !con)
 627                 return NULL;
 628
 629         if (head) {
 630                 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
 631                         return NULL;
 632
 633                 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
 634                         if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
 635                                 return NULL;
 636
 637                         obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
 638                 } else
 639                         obj = &con->objs[head->block];
 640
 641                 if (alive_obj(obj))
 642                         return obj;
 643         } else {
 644                 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
 645                         obj = &con->objs[i];
 646                         if (alive_obj(obj))
 647                                 return obj;
 648                 }
 649         }
 650
 651         return NULL;
 652 }
 653 /* obj end */
 654
 655 /* feature ctl begin */
 656 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
 657                                          struct ras_common_if *head)
 658 {
 659         return adev->ras_hw_enabled & BIT(head->block);
 660 }
 661
 662 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
 663                 struct ras_common_if *head)
 664 {
 665         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 666
 667         return con->features & BIT(head->block);
 668 }
 669
 670 /*
 671  * if obj is not created, then create one.
 672  * set feature enable flag.
 673  */
 674 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
 675                 struct ras_common_if *head, int enable)
 676 {
 677         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 678         struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
 679
 680         /* If hardware does not support ras, then do not create obj.
 681          * But if hardware support ras, we can create the obj.
 682          * Ras framework checks con->hw_supported to see if it need do
 683          * corresponding initialization.
 684          * IP checks con->support to see if it need disable ras.
 685          */
 686         if (!amdgpu_ras_is_feature_allowed(adev, head))
 687                 return 0;
 688
 689         if (enable) {
 690                 if (!obj) {
 691                         obj = amdgpu_ras_create_obj(adev, head);
 692                         if (!obj)
 693                                 return -EINVAL;
 694                 } else {
 695                         /* In case we create obj somewhere else */
 696                         get_obj(obj);
 697                 }
 698                 con->features |= BIT(head->block);
 699         } else {
 700                 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
 701                         con->features &= ~BIT(head->block);
 702                         put_obj(obj);
 703                 }
 704         }
 705
 706         return 0;
 707 }
 708
 709 /* wrapper of psp_ras_enable_features */
 710 int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
 711                 struct ras_common_if *head, bool enable)
 712 {
 713         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 714         union ta_ras_cmd_input *info;
 715         int ret;
 716
 717         if (!con)
 718                 return -EINVAL;
 719
 720         if (head->block == AMDGPU_RAS_BLOCK__GFX) {
 721                 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
 722                 if (!info)
 723                         return -ENOMEM;
 724
 725                 if (!enable) {
 726                         info->disable_features = (struct ta_ras_disable_features_input) {
 727                                 .block_id =  amdgpu_ras_block_to_ta(head->block),
 728                                 .error_type = amdgpu_ras_error_to_ta(head->type),
 729                         };
 730                 } else {
 731                         info->enable_features = (struct ta_ras_enable_features_input) {
 732                                 .block_id =  amdgpu_ras_block_to_ta(head->block),
 733                                 .error_type = amdgpu_ras_error_to_ta(head->type),
 734                         };
 735                 }
 736         }
 737
 738         /* Do not enable if it is not allowed. */
 739         WARN_ON(enable && !amdgpu_ras_is_feature_allowed(adev, head));
 740
 741         /* Only enable ras feature operation handle on host side */
 742         if (head->block == AMDGPU_RAS_BLOCK__GFX &&
 743                 !amdgpu_sriov_vf(adev) &&
 744                 !amdgpu_ras_intr_triggered()) {
 745                 ret = psp_ras_enable_features(&adev->psp, info, enable);
 746                 if (ret) {
 747                         dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
 748                                 enable ? "enable":"disable",
 749                                 get_ras_block_str(head),
 750                                 amdgpu_ras_is_poison_mode_supported(adev), ret);
 751                         goto out;
 752                 }
 753         }
 754
 755         /* setup the obj */
 756         __amdgpu_ras_feature_enable(adev, head, enable);
 757         ret = 0;
 758 out:
 759         if (head->block == AMDGPU_RAS_BLOCK__GFX)
 760                 kfree(info);
 761         return ret;
 762 }
 763
 764 /* Only used in device probe stage and called only once. */
 765 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
 766                 struct ras_common_if *head, bool enable)
 767 {
 768         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 769         int ret;
 770
 771         if (!con)
 772                 return -EINVAL;
 773
 774         if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
 775                 if (enable) {
 776                         /* There is no harm to issue a ras TA cmd regardless of
 777                          * the currecnt ras state.
 778                          * If current state == target state, it will do nothing
 779                          * But sometimes it requests driver to reset and repost
 780                          * with error code -EAGAIN.
 781                          */
 782                         ret = amdgpu_ras_feature_enable(adev, head, 1);
 783                         /* With old ras TA, we might fail to enable ras.
 784                          * Log it and just setup the object.
 785                          * TODO need remove this WA in the future.
 786                          */
 787                         if (ret == -EINVAL) {
 788                                 ret = __amdgpu_ras_feature_enable(adev, head, 1);
 789                                 if (!ret)
 790                                         dev_info(adev->dev,
 791                                                 "RAS INFO: %s setup object\n",
 792                                                 get_ras_block_str(head));
 793                         }
 794                 } else {
 795                         /* setup the object then issue a ras TA disable cmd.*/
 796                         ret = __amdgpu_ras_feature_enable(adev, head, 1);
 797                         if (ret)
 798                                 return ret;
 799
 800                         /* gfx block ras dsiable cmd must send to ras-ta */
 801                         if (head->block == AMDGPU_RAS_BLOCK__GFX)
 802                                 con->features |= BIT(head->block);
 803
 804                         ret = amdgpu_ras_feature_enable(adev, head, 0);
 805
 806                         /* clean gfx block ras features flag */
 807                         if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
 808                                 con->features &= ~BIT(head->block);
 809                 }
 810         } else
 811                 ret = amdgpu_ras_feature_enable(adev, head, enable);
 812
 813         return ret;
 814 }
 815
 816 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
 817                 bool bypass)
 818 {
 819         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 820         struct ras_manager *obj, *tmp;
 821
 822         list_for_each_entry_safe(obj, tmp, &con->head, node) {
 823                 /* bypass psp.
 824                  * aka just release the obj and corresponding flags
 825                  */
 826                 if (bypass) {
 827                         if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
 828                                 break;
 829                 } else {
 830                         if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
 831                                 break;
 832                 }
 833         }
 834
 835         return con->features;
 836 }
 837
 838 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
 839                 bool bypass)
 840 {
 841         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 842         int i;
 843         const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
 844
 845         for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
 846                 struct ras_common_if head = {
 847                         .block = i,
 848                         .type = default_ras_type,
 849                         .sub_block_index = 0,
 850                 };
 851
 852                 if (i == AMDGPU_RAS_BLOCK__MCA)
 853                         continue;
 854
 855                 if (bypass) {
 856                         /*
 857                          * bypass psp. vbios enable ras for us.
 858                          * so just create the obj
 859                          */
 860                         if (__amdgpu_ras_feature_enable(adev, &head, 1))
 861                                 break;
 862                 } else {
 863                         if (amdgpu_ras_feature_enable(adev, &head, 1))
 864                                 break;
 865                 }
 866         }
 867
 868         for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
 869                 struct ras_common_if head = {
 870                         .block = AMDGPU_RAS_BLOCK__MCA,
 871                         .type = default_ras_type,
 872                         .sub_block_index = i,
 873                 };
 874
 875                 if (bypass) {
 876                         /*
 877                          * bypass psp. vbios enable ras for us.
 878                          * so just create the obj
 879                          */
 880                         if (__amdgpu_ras_feature_enable(adev, &head, 1))
 881                                 break;
 882                 } else {
 883                         if (amdgpu_ras_feature_enable(adev, &head, 1))
 884                                 break;
 885                 }
 886         }
 887
 888         return con->features;
 889 }
 890 /* feature ctl end */
 891
 892 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
 893                 enum amdgpu_ras_block block)
 894 {
 895         if (!block_obj)
 896                 return -EINVAL;
 897
 898         if (block_obj->ras_comm.block == block)
 899                 return 0;
 900
 901         return -EINVAL;
 902 }
 903
 904 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
 905                                         enum amdgpu_ras_block block, uint32_t sub_block_index)
 906 {
 907         struct amdgpu_ras_block_list *node, *tmp;
 908         struct amdgpu_ras_block_object *obj;
 909
 910         if (block >= AMDGPU_RAS_BLOCK__LAST)
 911                 return NULL;
 912
 913         if (!amdgpu_ras_is_supported(adev, block))
 914                 return NULL;
 915
 916         list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
 917                 if (!node->ras_obj) {
 918                         dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
 919                         continue;
 920                 }
 921
 922                 obj = node->ras_obj;
 923                 if (obj->ras_block_match) {
 924                         if (obj->ras_block_match(obj, block, sub_block_index) == 0)
 925                                 return obj;
 926                 } else {
 927                         if (amdgpu_ras_block_match_default(obj, block) == 0)
 928                                 return obj;
 929                 }
 930         }
 931
 932         return NULL;
 933 }
 934
 935 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
 936 {
 937         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 938         int ret = 0;
 939
 940         /*
 941          * choosing right query method according to
 942          * whether smu support query error information
 943          */
 944         ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
 945         if (ret == -EOPNOTSUPP) {
 946                 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
 947                         adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
 948                         adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
 949
 950                 /* umc query_ras_error_address is also responsible for clearing
 951                  * error status
 952                  */
 953                 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
 954                     adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
 955                         adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
 956         } else if (!ret) {
 957                 if (adev->umc.ras &&
 958                         adev->umc.ras->ecc_info_query_ras_error_count)
 959                         adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
 960
 961                 if (adev->umc.ras &&
 962                         adev->umc.ras->ecc_info_query_ras_error_address)
 963                         adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
 964         }
 965 }
 966
 967 /* query/inject/cure begin */
 968 int amdgpu_ras_query_error_status(struct amdgpu_device *adev,
 969                                   struct ras_query_if *info)
 970 {
 971         struct amdgpu_ras_block_object *block_obj = NULL;
 972         struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
 973         struct ras_err_data err_data = {0, 0, 0, NULL};
 974
 975         if (!obj)
 976                 return -EINVAL;
 977
 978         if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
 979                 amdgpu_ras_get_ecc_info(adev, &err_data);
 980         } else {
 981                 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
 982                 if (!block_obj || !block_obj->hw_ops)   {
 983                         dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
 984                                      get_ras_block_str(&info->head));
 985                         return -EINVAL;
 986                 }
 987
 988                 if (block_obj->hw_ops->query_ras_error_count)
 989                         block_obj->hw_ops->query_ras_error_count(adev, &err_data);
 990
 991                 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
 992                     (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
 993                     (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
 994                                 if (block_obj->hw_ops->query_ras_error_status)
 995                                         block_obj->hw_ops->query_ras_error_status(adev);
 996                         }
 997         }
 998
 999         obj->err_data.ue_count += err_data.ue_count;
1000         obj->err_data.ce_count += err_data.ce_count;
1001
1002         info->ue_count = obj->err_data.ue_count;
1003         info->ce_count = obj->err_data.ce_count;
1004
1005         if (err_data.ce_count) {
1006                 if (adev->smuio.funcs &&
1007                     adev->smuio.funcs->get_socket_id &&
1008                     adev->smuio.funcs->get_die_id) {
1009                         dev_info(adev->dev, "socket: %d, die: %d "
1010                                         "%ld correctable hardware errors "
1011                                         "detected in %s block, no user "
1012                                         "action is needed.\n",
1013                                         adev->smuio.funcs->get_socket_id(adev),
1014                                         adev->smuio.funcs->get_die_id(adev),
1015                                         obj->err_data.ce_count,
1016                                         get_ras_block_str(&info->head));
1017                 } else {
1018                         dev_info(adev->dev, "%ld correctable hardware errors "
1019                                         "detected in %s block, no user "
1020                                         "action is needed.\n",
1021                                         obj->err_data.ce_count,
1022                                         get_ras_block_str(&info->head));
1023                 }
1024         }
1025         if (err_data.ue_count) {
1026                 if (adev->smuio.funcs &&
1027                     adev->smuio.funcs->get_socket_id &&
1028                     adev->smuio.funcs->get_die_id) {
1029                         dev_info(adev->dev, "socket: %d, die: %d "
1030                                         "%ld uncorrectable hardware errors "
1031                                         "detected in %s block\n",
1032                                         adev->smuio.funcs->get_socket_id(adev),
1033                                         adev->smuio.funcs->get_die_id(adev),
1034                                         obj->err_data.ue_count,
1035                                         get_ras_block_str(&info->head));
1036                 } else {
1037                         dev_info(adev->dev, "%ld uncorrectable hardware errors "
1038                                         "detected in %s block\n",
1039                                         obj->err_data.ue_count,
1040                                         get_ras_block_str(&info->head));
1041                 }
1042         }
1043
1044         return 0;
1045 }
1046
1047 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1048                 enum amdgpu_ras_block block)
1049 {
1050         struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1051
1052         if (!amdgpu_ras_is_supported(adev, block))
1053                 return -EINVAL;
1054
1055         if (!block_obj || !block_obj->hw_ops)   {
1056                 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1057                              ras_block_str(block));
1058                 return -EINVAL;
1059         }
1060
1061         if (block_obj->hw_ops->reset_ras_error_count)
1062                 block_obj->hw_ops->reset_ras_error_count(adev);
1063
1064         if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1065             (block == AMDGPU_RAS_BLOCK__MMHUB)) {
1066                 if (block_obj->hw_ops->reset_ras_error_status)
1067                         block_obj->hw_ops->reset_ras_error_status(adev);
1068         }
1069
1070         return 0;
1071 }
1072
1073 /* wrapper of psp_ras_trigger_error */
1074 int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1075                 struct ras_inject_if *info)
1076 {
1077         struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1078         struct ta_ras_trigger_error_input block_info = {
1079                 .block_id =  amdgpu_ras_block_to_ta(info->head.block),
1080                 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
1081                 .sub_block_index = info->head.sub_block_index,
1082                 .address = info->address,
1083                 .value = info->value,
1084         };
1085         int ret = -EINVAL;
1086         struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1087                                                         info->head.block,
1088                                                         info->head.sub_block_index);
1089
1090         if (!obj)
1091                 return -EINVAL;
1092
1093         if (!block_obj || !block_obj->hw_ops)   {
1094                 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1095                              get_ras_block_str(&info->head));
1096                 return -EINVAL;
1097         }
1098
1099         /* Calculate XGMI relative offset */
1100         if (adev->gmc.xgmi.num_physical_nodes > 1) {
1101                 block_info.address =
1102                         amdgpu_xgmi_get_relative_phy_addr(adev,
1103                                                           block_info.address);
1104         }
1105
1106         if (info->head.block == AMDGPU_RAS_BLOCK__GFX) {
1107                 if (block_obj->hw_ops->ras_error_inject)
1108                         ret = block_obj->hw_ops->ras_error_inject(adev, info);
1109         } else {
1110                 /* If defined special ras_error_inject(e.g: xgmi), implement special ras_error_inject */
1111                 if (block_obj->hw_ops->ras_error_inject)
1112                         ret = block_obj->hw_ops->ras_error_inject(adev, &block_info);
1113                 else  /*If not defined .ras_error_inject, use default ras_error_inject*/
1114                         ret = psp_ras_trigger_error(&adev->psp, &block_info);
1115         }
1116
1117         if (ret)
1118                 dev_err(adev->dev, "ras inject %s failed %d\n",
1119                         get_ras_block_str(&info->head), ret);
1120
1121         return ret;
1122 }
1123
1124 /**
1125  * amdgpu_ras_query_error_count -- Get error counts of all IPs
1126  * @adev: pointer to AMD GPU device
1127  * @ce_count: pointer to an integer to be set to the count of correctible errors.
1128  * @ue_count: pointer to an integer to be set to the count of uncorrectible
1129  * errors.
1130  *
1131  * If set, @ce_count or @ue_count, count and return the corresponding
1132  * error counts in those integer pointers. Return 0 if the device
1133  * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1134  */
1135 int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1136                                  unsigned long *ce_count,
1137                                  unsigned long *ue_count)
1138 {
1139         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1140         struct ras_manager *obj;
1141         unsigned long ce, ue;
1142
1143         if (!adev->ras_enabled || !con)
1144                 return -EOPNOTSUPP;
1145
1146         /* Don't count since no reporting.
1147          */
1148         if (!ce_count && !ue_count)
1149                 return 0;
1150
1151         ce = 0;
1152         ue = 0;
1153         list_for_each_entry(obj, &con->head, node) {
1154                 struct ras_query_if info = {
1155                         .head = obj->head,
1156                 };
1157                 int res;
1158
1159                 res = amdgpu_ras_query_error_status(adev, &info);
1160                 if (res)
1161                         return res;
1162
1163                 if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
1164                     adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
1165                         if (amdgpu_ras_reset_error_status(adev, info.head.block))
1166                                 dev_warn(adev->dev, "Failed to reset error counter and error status");
1167                 }
1168
1169                 ce += info.ce_count;
1170                 ue += info.ue_count;
1171         }
1172
1173         if (ce_count)
1174                 *ce_count = ce;
1175
1176         if (ue_count)
1177                 *ue_count = ue;
1178
1179         return 0;
1180 }
1181 /* query/inject/cure end */
1182
1183
1184 /* sysfs begin */
1185
1186 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1187                 struct ras_badpage **bps, unsigned int *count);
1188
1189 static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1190 {
1191         switch (flags) {
1192         case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
1193                 return "R";
1194         case AMDGPU_RAS_RETIRE_PAGE_PENDING:
1195                 return "P";
1196         case AMDGPU_RAS_RETIRE_PAGE_FAULT:
1197         default:
1198                 return "F";
1199         }
1200 }
1201
1202 /**
1203  * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
1204  *
1205  * It allows user to read the bad pages of vram on the gpu through
1206  * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1207  *
1208  * It outputs multiple lines, and each line stands for one gpu page.
1209  *
1210  * The format of one line is below,
1211  * gpu pfn : gpu page size : flags
1212  *
1213  * gpu pfn and gpu page size are printed in hex format.
1214  * flags can be one of below character,
1215  *
1216  * R: reserved, this gpu page is reserved and not able to use.
1217  *
1218  * P: pending for reserve, this gpu page is marked as bad, will be reserved
1219  * in next window of page_reserve.
1220  *
1221  * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1222  *
1223  * Examples:
1224  *
1225  * .. code-block:: bash
1226  *
1227  *      0x00000001 : 0x00001000 : R
1228  *      0x00000002 : 0x00001000 : P
1229  *
1230  */
1231
1232 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1233                 struct kobject *kobj, struct bin_attribute *attr,
1234                 char *buf, loff_t ppos, size_t count)
1235 {
1236         struct amdgpu_ras *con =
1237                 container_of(attr, struct amdgpu_ras, badpages_attr);
1238         struct amdgpu_device *adev = con->adev;
1239         const unsigned int element_size =
1240                 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
1241         unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1242         unsigned int end = div64_ul(ppos + count - 1, element_size);
1243         ssize_t s = 0;
1244         struct ras_badpage *bps = NULL;
1245         unsigned int bps_count = 0;
1246
1247         memset(buf, 0, count);
1248
1249         if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1250                 return 0;
1251
1252         for (; start < end && start < bps_count; start++)
1253                 s += scnprintf(&buf[s], element_size + 1,
1254                                 "0x%08x : 0x%08x : %1s\n",
1255                                 bps[start].bp,
1256                                 bps[start].size,
1257                                 amdgpu_ras_badpage_flags_str(bps[start].flags));
1258
1259         kfree(bps);
1260
1261         return s;
1262 }
1263
1264 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1265                 struct device_attribute *attr, char *buf)
1266 {
1267         struct amdgpu_ras *con =
1268                 container_of(attr, struct amdgpu_ras, features_attr);
1269
1270         return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
1271 }
1272
1273 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1274 {
1275         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1276
1277         sysfs_remove_file_from_group(&adev->dev->kobj,
1278                                 &con->badpages_attr.attr,
1279                                 RAS_FS_NAME);
1280 }
1281
1282 static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev)
1283 {
1284         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1285         struct attribute *attrs[] = {
1286                 &con->features_attr.attr,
1287                 NULL
1288         };
1289         struct attribute_group group = {
1290                 .name = RAS_FS_NAME,
1291                 .attrs = attrs,
1292         };
1293
1294         sysfs_remove_group(&adev->dev->kobj, &group);
1295
1296         return 0;
1297 }
1298
1299 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
1300                 struct ras_common_if *head)
1301 {
1302         struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1303
1304         if (!obj || obj->attr_inuse)
1305                 return -EINVAL;
1306
1307         get_obj(obj);
1308
1309         snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1310                 "%s_err_count", head->name);
1311
1312         obj->sysfs_attr = (struct device_attribute){
1313                 .attr = {
1314                         .name = obj->fs_data.sysfs_name,
1315                         .mode = S_IRUGO,
1316                 },
1317                         .show = amdgpu_ras_sysfs_read,
1318         };
1319         sysfs_attr_init(&obj->sysfs_attr.attr);
1320
1321         if (sysfs_add_file_to_group(&adev->dev->kobj,
1322                                 &obj->sysfs_attr.attr,
1323                                 RAS_FS_NAME)) {
1324                 put_obj(obj);
1325                 return -EINVAL;
1326         }
1327
1328         obj->attr_inuse = 1;
1329
1330         return 0;
1331 }
1332
1333 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1334                 struct ras_common_if *head)
1335 {
1336         struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1337
1338         if (!obj || !obj->attr_inuse)
1339                 return -EINVAL;
1340
1341         sysfs_remove_file_from_group(&adev->dev->kobj,
1342                                 &obj->sysfs_attr.attr,
1343                                 RAS_FS_NAME);
1344         obj->attr_inuse = 0;
1345         put_obj(obj);
1346
1347         return 0;
1348 }
1349
1350 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1351 {
1352         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1353         struct ras_manager *obj, *tmp;
1354
1355         list_for_each_entry_safe(obj, tmp, &con->head, node) {
1356                 amdgpu_ras_sysfs_remove(adev, &obj->head);
1357         }
1358
1359         if (amdgpu_bad_page_threshold != 0)
1360                 amdgpu_ras_sysfs_remove_bad_page_node(adev);
1361
1362         amdgpu_ras_sysfs_remove_feature_node(adev);
1363
1364         return 0;
1365 }
1366 /* sysfs end */
1367
1368 /**
1369  * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1370  *
1371  * Normally when there is an uncorrectable error, the driver will reset
1372  * the GPU to recover.  However, in the event of an unrecoverable error,
1373  * the driver provides an interface to reboot the system automatically
1374  * in that event.
1375  *
1376  * The following file in debugfs provides that interface:
1377  * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1378  *
1379  * Usage:
1380  *
1381  * .. code-block:: bash
1382  *
1383  *      echo true > .../ras/auto_reboot
1384  *
1385  */
1386 /* debugfs begin */
1387 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
1388 {
1389         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1390         struct drm_minor  *minor = adev_to_drm(adev)->primary;
1391         struct dentry     *dir;
1392
1393         dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1394         debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1395                             &amdgpu_ras_debugfs_ctrl_ops);
1396         debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1397                             &amdgpu_ras_debugfs_eeprom_ops);
1398         debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1399                            &con->bad_page_cnt_threshold);
1400         debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1401         debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1402         debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1403                             &amdgpu_ras_debugfs_eeprom_size_ops);
1404         con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1405                                                        S_IRUGO, dir, adev,
1406                                                        &amdgpu_ras_debugfs_eeprom_table_ops);
1407         amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
1408
1409         /*
1410          * After one uncorrectable error happens, usually GPU recovery will
1411          * be scheduled. But due to the known problem in GPU recovery failing
1412          * to bring GPU back, below interface provides one direct way to
1413          * user to reboot system automatically in such case within
1414          * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1415          * will never be called.
1416          */
1417         debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
1418
1419         /*
1420          * User could set this not to clean up hardware's error count register
1421          * of RAS IPs during ras recovery.
1422          */
1423         debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1424                             &con->disable_ras_err_cnt_harvest);
1425         return dir;
1426 }
1427
1428 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
1429                                       struct ras_fs_if *head,
1430                                       struct dentry *dir)
1431 {
1432         struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
1433
1434         if (!obj || !dir)
1435                 return;
1436
1437         get_obj(obj);
1438
1439         memcpy(obj->fs_data.debugfs_name,
1440                         head->debugfs_name,
1441                         sizeof(obj->fs_data.debugfs_name));
1442
1443         debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
1444                             obj, &amdgpu_ras_debugfs_ops);
1445 }
1446
1447 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
1448 {
1449         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1450         struct dentry *dir;
1451         struct ras_manager *obj;
1452         struct ras_fs_if fs_info;
1453
1454         /*
1455          * it won't be called in resume path, no need to check
1456          * suspend and gpu reset status
1457          */
1458         if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
1459                 return;
1460
1461         dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
1462
1463         list_for_each_entry(obj, &con->head, node) {
1464                 if (amdgpu_ras_is_supported(adev, obj->head.block) &&
1465                         (obj->attr_inuse == 1)) {
1466                         sprintf(fs_info.debugfs_name, "%s_err_inject",
1467                                         get_ras_block_str(&obj->head));
1468                         fs_info.head = obj->head;
1469                         amdgpu_ras_debugfs_create(adev, &fs_info, dir);
1470                 }
1471         }
1472 }
1473
1474 /* debugfs end */
1475
1476 /* ras fs */
1477 static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
1478                 amdgpu_ras_sysfs_badpages_read, NULL, 0);
1479 static DEVICE_ATTR(features, S_IRUGO,
1480                 amdgpu_ras_sysfs_features_read, NULL);
1481 static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
1482 {
1483         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1484         struct attribute_group group = {
1485                 .name = RAS_FS_NAME,
1486         };
1487         struct attribute *attrs[] = {
1488                 &con->features_attr.attr,
1489                 NULL
1490         };
1491         struct bin_attribute *bin_attrs[] = {
1492                 NULL,
1493                 NULL,
1494         };
1495         int r;
1496
1497         /* add features entry */
1498         con->features_attr = dev_attr_features;
1499         group.attrs = attrs;
1500         sysfs_attr_init(attrs[0]);
1501
1502         if (amdgpu_bad_page_threshold != 0) {
1503                 /* add bad_page_features entry */
1504                 bin_attr_gpu_vram_bad_pages.private = NULL;
1505                 con->badpages_attr = bin_attr_gpu_vram_bad_pages;
1506                 bin_attrs[0] = &con->badpages_attr;
1507                 group.bin_attrs = bin_attrs;
1508                 sysfs_bin_attr_init(bin_attrs[0]);
1509         }
1510
1511         r = sysfs_create_group(&adev->dev->kobj, &group);
1512         if (r)
1513                 dev_err(adev->dev, "Failed to create RAS sysfs group!");
1514
1515         return 0;
1516 }
1517
1518 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
1519 {
1520         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1521         struct ras_manager *con_obj, *ip_obj, *tmp;
1522
1523         if (IS_ENABLED(CONFIG_DEBUG_FS)) {
1524                 list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
1525                         ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
1526                         if (ip_obj)
1527                                 put_obj(ip_obj);
1528                 }
1529         }
1530
1531         amdgpu_ras_sysfs_remove_all(adev);
1532         return 0;
1533 }
1534 /* ras fs end */
1535
1536 /* ih begin */
1537
1538 /* For the hardware that cannot enable bif ring for both ras_controller_irq
1539  * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
1540  * register to check whether the interrupt is triggered or not, and properly
1541  * ack the interrupt if it is there
1542  */
1543 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
1544 {
1545         /* Fatal error events are handled on host side */
1546         if (amdgpu_sriov_vf(adev) ||
1547                 !amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__PCIE_BIF))
1548                 return;
1549
1550         if (adev->nbio.ras &&
1551             adev->nbio.ras->handle_ras_controller_intr_no_bifring)
1552                 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
1553
1554         if (adev->nbio.ras &&
1555             adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
1556                 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
1557 }
1558
1559 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
1560                                 struct amdgpu_iv_entry *entry)
1561 {
1562         bool poison_stat = false;
1563         struct amdgpu_device *adev = obj->adev;
1564         struct amdgpu_ras_block_object *block_obj =
1565                 amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
1566
1567         if (!block_obj || !block_obj->hw_ops)
1568                 return;
1569
1570         /* both query_poison_status and handle_poison_consumption are optional,
1571          * but at least one of them should be implemented if we need poison
1572          * consumption handler
1573          */
1574         if (block_obj->hw_ops->query_poison_status) {
1575                 poison_stat = block_obj->hw_ops->query_poison_status(adev);
1576                 if (!poison_stat) {
1577                         /* Not poison consumption interrupt, no need to handle it */
1578                         dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
1579                                         block_obj->ras_comm.name);
1580
1581                         return;
1582                 }
1583         }
1584
1585         if (!adev->gmc.xgmi.connected_to_cpu)
1586                 amdgpu_umc_poison_handler(adev, false);
1587
1588         if (block_obj->hw_ops->handle_poison_consumption)
1589                 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
1590
1591         /* gpu reset is fallback for failed and default cases */
1592         if (poison_stat) {
1593                 dev_info(adev->dev, "GPU reset for %s RAS poison consumption is issued!\n",
1594                                 block_obj->ras_comm.name);
1595                 amdgpu_ras_reset_gpu(adev);
1596         }
1597 }
1598
1599 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
1600                                 struct amdgpu_iv_entry *entry)
1601 {
1602         dev_info(obj->adev->dev,
1603                 "Poison is created, no user action is needed.\n");
1604 }
1605
1606 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
1607                                 struct amdgpu_iv_entry *entry)
1608 {
1609         struct ras_ih_data *data = &obj->ih_data;
1610         struct ras_err_data err_data = {0, 0, 0, NULL};
1611         int ret;
1612
1613         if (!data->cb)
1614                 return;
1615
1616         /* Let IP handle its data, maybe we need get the output
1617          * from the callback to update the error type/count, etc
1618          */
1619         ret = data->cb(obj->adev, &err_data, entry);
1620         /* ue will trigger an interrupt, and in that case
1621          * we need do a reset to recovery the whole system.
1622          * But leave IP do that recovery, here we just dispatch
1623          * the error.
1624          */
1625         if (ret == AMDGPU_RAS_SUCCESS) {
1626                 /* these counts could be left as 0 if
1627                  * some blocks do not count error number
1628                  */
1629                 obj->err_data.ue_count += err_data.ue_count;
1630                 obj->err_data.ce_count += err_data.ce_count;
1631         }
1632 }
1633
1634 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
1635 {
1636         struct ras_ih_data *data = &obj->ih_data;
1637         struct amdgpu_iv_entry entry;
1638
1639         while (data->rptr != data->wptr) {
1640                 rmb();
1641                 memcpy(&entry, &data->ring[data->rptr],
1642                                 data->element_size);
1643
1644                 wmb();
1645                 data->rptr = (data->aligned_element_size +
1646                                 data->rptr) % data->ring_size;
1647
1648                 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
1649                         if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1650                                 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
1651                         else
1652                                 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
1653                 } else {
1654                         if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1655                                 amdgpu_ras_interrupt_umc_handler(obj, &entry);
1656                         else
1657                                 dev_warn(obj->adev->dev,
1658                                         "No RAS interrupt handler for non-UMC block with poison disabled.\n");
1659                 }
1660         }
1661 }
1662
1663 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
1664 {
1665         struct ras_ih_data *data =
1666                 container_of(work, struct ras_ih_data, ih_work);
1667         struct ras_manager *obj =
1668                 container_of(data, struct ras_manager, ih_data);
1669
1670         amdgpu_ras_interrupt_handler(obj);
1671 }
1672
1673 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
1674                 struct ras_dispatch_if *info)
1675 {
1676         struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1677         struct ras_ih_data *data = &obj->ih_data;
1678
1679         if (!obj)
1680                 return -EINVAL;
1681
1682         if (data->inuse == 0)
1683                 return 0;
1684
1685         /* Might be overflow... */
1686         memcpy(&data->ring[data->wptr], info->entry,
1687                         data->element_size);
1688
1689         wmb();
1690         data->wptr = (data->aligned_element_size +
1691                         data->wptr) % data->ring_size;
1692
1693         schedule_work(&data->ih_work);
1694
1695         return 0;
1696 }
1697
1698 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
1699                 struct ras_common_if *head)
1700 {
1701         struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1702         struct ras_ih_data *data;
1703
1704         if (!obj)
1705                 return -EINVAL;
1706
1707         data = &obj->ih_data;
1708         if (data->inuse == 0)
1709                 return 0;
1710
1711         cancel_work_sync(&data->ih_work);
1712
1713         kfree(data->ring);
1714         memset(data, 0, sizeof(*data));
1715         put_obj(obj);
1716
1717         return 0;
1718 }
1719
1720 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
1721                 struct ras_common_if *head)
1722 {
1723         struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1724         struct ras_ih_data *data;
1725         struct amdgpu_ras_block_object *ras_obj;
1726
1727         if (!obj) {
1728                 /* in case we registe the IH before enable ras feature */
1729                 obj = amdgpu_ras_create_obj(adev, head);
1730                 if (!obj)
1731                         return -EINVAL;
1732         } else
1733                 get_obj(obj);
1734
1735         ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
1736
1737         data = &obj->ih_data;
1738         /* add the callback.etc */
1739         *data = (struct ras_ih_data) {
1740                 .inuse = 0,
1741                 .cb = ras_obj->ras_cb,
1742                 .element_size = sizeof(struct amdgpu_iv_entry),
1743                 .rptr = 0,
1744                 .wptr = 0,
1745         };
1746
1747         INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
1748
1749         data->aligned_element_size = ALIGN(data->element_size, 8);
1750         /* the ring can store 64 iv entries. */
1751         data->ring_size = 64 * data->aligned_element_size;
1752         data->ring = kmalloc(data->ring_size, GFP_KERNEL);
1753         if (!data->ring) {
1754                 put_obj(obj);
1755                 return -ENOMEM;
1756         }
1757
1758         /* IH is ready */
1759         data->inuse = 1;
1760
1761         return 0;
1762 }
1763
1764 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
1765 {
1766         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1767         struct ras_manager *obj, *tmp;
1768
1769         list_for_each_entry_safe(obj, tmp, &con->head, node) {
1770                 amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
1771         }
1772
1773         return 0;
1774 }
1775 /* ih end */
1776
1777 /* traversal all IPs except NBIO to query error counter */
1778 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev)
1779 {
1780         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1781         struct ras_manager *obj;
1782
1783         if (!adev->ras_enabled || !con)
1784                 return;
1785
1786         list_for_each_entry(obj, &con->head, node) {
1787                 struct ras_query_if info = {
1788                         .head = obj->head,
1789                 };
1790
1791                 /*
1792                  * PCIE_BIF IP has one different isr by ras controller
1793                  * interrupt, the specific ras counter query will be
1794                  * done in that isr. So skip such block from common
1795                  * sync flood interrupt isr calling.
1796                  */
1797                 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
1798                         continue;
1799
1800                 /*
1801                  * this is a workaround for aldebaran, skip send msg to
1802                  * smu to get ecc_info table due to smu handle get ecc
1803                  * info table failed temporarily.
1804                  * should be removed until smu fix handle ecc_info table.
1805                  */
1806                 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
1807                         (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2)))
1808                         continue;
1809
1810                 amdgpu_ras_query_error_status(adev, &info);
1811
1812                 if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
1813                     adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4) &&
1814                     adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 0)) {
1815                         if (amdgpu_ras_reset_error_status(adev, info.head.block))
1816                                 dev_warn(adev->dev, "Failed to reset error counter and error status");
1817                 }
1818         }
1819 }
1820
1821 /* Parse RdRspStatus and WrRspStatus */
1822 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
1823                                           struct ras_query_if *info)
1824 {
1825         struct amdgpu_ras_block_object *block_obj;
1826         /*
1827          * Only two block need to query read/write
1828          * RspStatus at current state
1829          */
1830         if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
1831                 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
1832                 return;
1833
1834         block_obj = amdgpu_ras_get_ras_block(adev,
1835                                         info->head.block,
1836                                         info->head.sub_block_index);
1837
1838         if (!block_obj || !block_obj->hw_ops) {
1839                 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1840                              get_ras_block_str(&info->head));
1841                 return;
1842         }
1843
1844         if (block_obj->hw_ops->query_ras_error_status)
1845                 block_obj->hw_ops->query_ras_error_status(adev);
1846
1847 }
1848
1849 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
1850 {
1851         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1852         struct ras_manager *obj;
1853
1854         if (!adev->ras_enabled || !con)
1855                 return;
1856
1857         list_for_each_entry(obj, &con->head, node) {
1858                 struct ras_query_if info = {
1859                         .head = obj->head,
1860                 };
1861
1862                 amdgpu_ras_error_status_query(adev, &info);
1863         }
1864 }
1865
1866 /* recovery begin */
1867
1868 /* return 0 on success.
1869  * caller need free bps.
1870  */
1871 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1872                 struct ras_badpage **bps, unsigned int *count)
1873 {
1874         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1875         struct ras_err_handler_data *data;
1876         int i = 0;
1877         int ret = 0, status;
1878
1879         if (!con || !con->eh_data || !bps || !count)
1880                 return -EINVAL;
1881
1882         mutex_lock(&con->recovery_lock);
1883         data = con->eh_data;
1884         if (!data || data->count == 0) {
1885                 *bps = NULL;
1886                 ret = -EINVAL;
1887                 goto out;
1888         }
1889
1890         *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
1891         if (!*bps) {
1892                 ret = -ENOMEM;
1893                 goto out;
1894         }
1895
1896         for (; i < data->count; i++) {
1897                 (*bps)[i] = (struct ras_badpage){
1898                         .bp = data->bps[i].retired_page,
1899                         .size = AMDGPU_GPU_PAGE_SIZE,
1900                         .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
1901                 };
1902                 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
1903                                 data->bps[i].retired_page);
1904                 if (status == -EBUSY)
1905                         (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
1906                 else if (status == -ENOENT)
1907                         (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
1908         }
1909
1910         *count = data->count;
1911 out:
1912         mutex_unlock(&con->recovery_lock);
1913         return ret;
1914 }
1915
1916 static void amdgpu_ras_do_recovery(struct work_struct *work)
1917 {
1918         struct amdgpu_ras *ras =
1919                 container_of(work, struct amdgpu_ras, recovery_work);
1920         struct amdgpu_device *remote_adev = NULL;
1921         struct amdgpu_device *adev = ras->adev;
1922         struct list_head device_list, *device_list_handle =  NULL;
1923
1924         if (!ras->disable_ras_err_cnt_harvest) {
1925                 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
1926
1927                 /* Build list of devices to query RAS related errors */
1928                 if  (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
1929                         device_list_handle = &hive->device_list;
1930                 } else {
1931                         INIT_LIST_HEAD(&device_list);
1932                         list_add_tail(&adev->gmc.xgmi.head, &device_list);
1933                         device_list_handle = &device_list;
1934                 }
1935
1936                 list_for_each_entry(remote_adev,
1937                                 device_list_handle, gmc.xgmi.head) {
1938                         amdgpu_ras_query_err_status(remote_adev);
1939                         amdgpu_ras_log_on_err_counter(remote_adev);
1940                 }
1941
1942                 amdgpu_put_xgmi_hive(hive);
1943         }
1944
1945         if (amdgpu_device_should_recover_gpu(ras->adev)) {
1946                 struct amdgpu_reset_context reset_context;
1947                 memset(&reset_context, 0, sizeof(reset_context));
1948
1949                 reset_context.method = AMD_RESET_METHOD_NONE;
1950                 reset_context.reset_req_dev = adev;
1951
1952                 /* Perform full reset in fatal error mode */
1953                 if (!amdgpu_ras_is_poison_mode_supported(ras->adev))
1954                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1955                 else
1956                         clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1957
1958                 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
1959         }
1960         atomic_set(&ras->in_recovery, 0);
1961 }
1962
1963 /* alloc/realloc bps array */
1964 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
1965                 struct ras_err_handler_data *data, int pages)
1966 {
1967         unsigned int old_space = data->count + data->space_left;
1968         unsigned int new_space = old_space + pages;
1969         unsigned int align_space = ALIGN(new_space, 512);
1970         void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
1971
1972         if (!bps) {
1973                 return -ENOMEM;
1974         }
1975
1976         if (data->bps) {
1977                 memcpy(bps, data->bps,
1978                                 data->count * sizeof(*data->bps));
1979                 kfree(data->bps);
1980         }
1981
1982         data->bps = bps;
1983         data->space_left += align_space - old_space;
1984         return 0;
1985 }
1986
1987 /* it deal with vram only. */
1988 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
1989                 struct eeprom_table_record *bps, int pages)
1990 {
1991         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1992         struct ras_err_handler_data *data;
1993         int ret = 0;
1994         uint32_t i;
1995
1996         if (!con || !con->eh_data || !bps || pages <= 0)
1997                 return 0;
1998
1999         mutex_lock(&con->recovery_lock);
2000         data = con->eh_data;
2001         if (!data)
2002                 goto out;
2003
2004         for (i = 0; i < pages; i++) {
2005                 if (amdgpu_ras_check_bad_page_unlock(con,
2006                         bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2007                         continue;
2008
2009                 if (!data->space_left &&
2010                         amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
2011                         ret = -ENOMEM;
2012                         goto out;
2013                 }
2014
2015                 amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr,
2016                         bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT,
2017                         AMDGPU_GPU_PAGE_SIZE);
2018
2019                 memcpy(&data->bps[data->count], &bps[i], sizeof(*data->bps));
2020                 data->count++;
2021                 data->space_left--;
2022         }
2023 out:
2024         mutex_unlock(&con->recovery_lock);
2025
2026         return ret;
2027 }
2028
2029 /*
2030  * write error record array to eeprom, the function should be
2031  * protected by recovery_lock
2032  */
2033 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev)
2034 {
2035         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2036         struct ras_err_handler_data *data;
2037         struct amdgpu_ras_eeprom_control *control;
2038         int save_count;
2039
2040         if (!con || !con->eh_data)
2041                 return 0;
2042
2043         mutex_lock(&con->recovery_lock);
2044         control = &con->eeprom_control;
2045         data = con->eh_data;
2046         save_count = data->count - control->ras_num_recs;
2047         mutex_unlock(&con->recovery_lock);
2048         /* only new entries are saved */
2049         if (save_count > 0) {
2050                 if (amdgpu_ras_eeprom_append(control,
2051                                              &data->bps[control->ras_num_recs],
2052                                              save_count)) {
2053                         dev_err(adev->dev, "Failed to save EEPROM table data!");
2054                         return -EIO;
2055                 }
2056
2057                 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
2058         }
2059
2060         return 0;
2061 }
2062
2063 /*
2064  * read error record array in eeprom and reserve enough space for
2065  * storing new bad pages
2066  */
2067 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
2068 {
2069         struct amdgpu_ras_eeprom_control *control =
2070                 &adev->psp.ras_context.ras->eeprom_control;
2071         struct eeprom_table_record *bps;
2072         int ret;
2073
2074         /* no bad page record, skip eeprom access */
2075         if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
2076                 return 0;
2077
2078         bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
2079         if (!bps)
2080                 return -ENOMEM;
2081
2082         ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
2083         if (ret)
2084                 dev_err(adev->dev, "Failed to load EEPROM table records!");
2085         else
2086                 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs);
2087
2088         kfree(bps);
2089         return ret;
2090 }
2091
2092 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
2093                                 uint64_t addr)
2094 {
2095         struct ras_err_handler_data *data = con->eh_data;
2096         int i;
2097
2098         addr >>= AMDGPU_GPU_PAGE_SHIFT;
2099         for (i = 0; i < data->count; i++)
2100                 if (addr == data->bps[i].retired_page)
2101                         return true;
2102
2103         return false;
2104 }
2105
2106 /*
2107  * check if an address belongs to bad page
2108  *
2109  * Note: this check is only for umc block
2110  */
2111 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
2112                                 uint64_t addr)
2113 {
2114         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2115         bool ret = false;
2116
2117         if (!con || !con->eh_data)
2118                 return ret;
2119
2120         mutex_lock(&con->recovery_lock);
2121         ret = amdgpu_ras_check_bad_page_unlock(con, addr);
2122         mutex_unlock(&con->recovery_lock);
2123         return ret;
2124 }
2125
2126 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
2127                                           uint32_t max_count)
2128 {
2129         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2130
2131         /*
2132          * Justification of value bad_page_cnt_threshold in ras structure
2133          *
2134          * Generally, -1 <= amdgpu_bad_page_threshold <= max record length
2135          * in eeprom, and introduce two scenarios accordingly.
2136          *
2137          * Bad page retirement enablement:
2138          *    - If amdgpu_bad_page_threshold = -1,
2139          *      bad_page_cnt_threshold = typical value by formula.
2140          *
2141          *    - When the value from user is 0 < amdgpu_bad_page_threshold <
2142          *      max record length in eeprom, use it directly.
2143          *
2144          * Bad page retirement disablement:
2145          *    - If amdgpu_bad_page_threshold = 0, bad page retirement
2146          *      functionality is disabled, and bad_page_cnt_threshold will
2147          *      take no effect.
2148          */
2149
2150         if (amdgpu_bad_page_threshold < 0) {
2151                 u64 val = adev->gmc.mc_vram_size;
2152
2153                 do_div(val, RAS_BAD_PAGE_COVER);
2154                 con->bad_page_cnt_threshold = min(lower_32_bits(val),
2155                                                   max_count);
2156         } else {
2157                 con->bad_page_cnt_threshold = min_t(int, max_count,
2158                                                     amdgpu_bad_page_threshold);
2159         }
2160 }
2161
2162 int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
2163 {
2164         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2165         struct ras_err_handler_data **data;
2166         u32  max_eeprom_records_count = 0;
2167         bool exc_err_limit = false;
2168         int ret;
2169
2170         if (!con || amdgpu_sriov_vf(adev))
2171                 return 0;
2172
2173         /* Allow access to RAS EEPROM via debugfs, when the ASIC
2174          * supports RAS and debugfs is enabled, but when
2175          * adev->ras_enabled is unset, i.e. when "ras_enable"
2176          * module parameter is set to 0.
2177          */
2178         con->adev = adev;
2179
2180         if (!adev->ras_enabled)
2181                 return 0;
2182
2183         data = &con->eh_data;
2184         *data = kmalloc(sizeof(**data), GFP_KERNEL | __GFP_ZERO);
2185         if (!*data) {
2186                 ret = -ENOMEM;
2187                 goto out;
2188         }
2189
2190         mutex_init(&con->recovery_lock);
2191         INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
2192         atomic_set(&con->in_recovery, 0);
2193         con->eeprom_control.bad_channel_bitmap = 0;
2194
2195         max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count();
2196         amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
2197
2198         /* Todo: During test the SMU might fail to read the eeprom through I2C
2199          * when the GPU is pending on XGMI reset during probe time
2200          * (Mostly after second bus reset), skip it now
2201          */
2202         if (adev->gmc.xgmi.pending_reset)
2203                 return 0;
2204         ret = amdgpu_ras_eeprom_init(&con->eeprom_control, &exc_err_limit);
2205         /*
2206          * This calling fails when exc_err_limit is true or
2207          * ret != 0.
2208          */
2209         if (exc_err_limit || ret)
2210                 goto free;
2211
2212         if (con->eeprom_control.ras_num_recs) {
2213                 ret = amdgpu_ras_load_bad_pages(adev);
2214                 if (ret)
2215                         goto free;
2216
2217                 amdgpu_dpm_send_hbm_bad_pages_num(adev, con->eeprom_control.ras_num_recs);
2218
2219                 if (con->update_channel_flag == true) {
2220                         amdgpu_dpm_send_hbm_bad_channel_flag(adev, con->eeprom_control.bad_channel_bitmap);
2221                         con->update_channel_flag = false;
2222                 }
2223         }
2224
2225 #ifdef CONFIG_X86_MCE_AMD
2226         if ((adev->asic_type == CHIP_ALDEBARAN) &&
2227             (adev->gmc.xgmi.connected_to_cpu))
2228                 amdgpu_register_bad_pages_mca_notifier(adev);
2229 #endif
2230         return 0;
2231
2232 free:
2233         kfree((*data)->bps);
2234         kfree(*data);
2235         con->eh_data = NULL;
2236 out:
2237         dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
2238
2239         /*
2240          * Except error threshold exceeding case, other failure cases in this
2241          * function would not fail amdgpu driver init.
2242          */
2243         if (!exc_err_limit)
2244                 ret = 0;
2245         else
2246                 ret = -EINVAL;
2247
2248         return ret;
2249 }
2250
2251 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
2252 {
2253         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2254         struct ras_err_handler_data *data = con->eh_data;
2255
2256         /* recovery_init failed to init it, fini is useless */
2257         if (!data)
2258                 return 0;
2259
2260         cancel_work_sync(&con->recovery_work);
2261
2262         mutex_lock(&con->recovery_lock);
2263         con->eh_data = NULL;
2264         kfree(data->bps);
2265         kfree(data);
2266         mutex_unlock(&con->recovery_lock);
2267
2268         return 0;
2269 }
2270 /* recovery end */
2271
2272 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
2273 {
2274         if (amdgpu_sriov_vf(adev)) {
2275                 switch (adev->ip_versions[MP0_HWIP][0]) {
2276                 case IP_VERSION(13, 0, 2):
2277                         return true;
2278                 default:
2279                         return false;
2280                 }
2281         }
2282
2283         if (adev->asic_type == CHIP_IP_DISCOVERY) {
2284                 switch (adev->ip_versions[MP0_HWIP][0]) {
2285                 case IP_VERSION(13, 0, 0):
2286                 case IP_VERSION(13, 0, 10):
2287                         return true;
2288                 default:
2289                         return false;
2290                 }
2291         }
2292
2293         return adev->asic_type == CHIP_VEGA10 ||
2294                 adev->asic_type == CHIP_VEGA20 ||
2295                 adev->asic_type == CHIP_ARCTURUS ||
2296                 adev->asic_type == CHIP_ALDEBARAN ||
2297                 adev->asic_type == CHIP_SIENNA_CICHLID;
2298 }
2299
2300 /*
2301  * this is workaround for vega20 workstation sku,
2302  * force enable gfx ras, ignore vbios gfx ras flag
2303  * due to GC EDC can not write
2304  */
2305 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
2306 {
2307         struct atom_context *ctx = adev->mode_info.atom_context;
2308
2309         if (!ctx)
2310                 return;
2311
2312         if (strnstr(ctx->vbios_version, "D16406",
2313                     sizeof(ctx->vbios_version)) ||
2314                 strnstr(ctx->vbios_version, "D36002",
2315                         sizeof(ctx->vbios_version)))
2316                 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
2317 }
2318
2319 /*
2320  * check hardware's ras ability which will be saved in hw_supported.
2321  * if hardware does not support ras, we can skip some ras initializtion and
2322  * forbid some ras operations from IP.
2323  * if software itself, say boot parameter, limit the ras ability. We still
2324  * need allow IP do some limited operations, like disable. In such case,
2325  * we have to initialize ras as normal. but need check if operation is
2326  * allowed or not in each function.
2327  */
2328 static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
2329 {
2330         adev->ras_hw_enabled = adev->ras_enabled = 0;
2331
2332         if (!adev->is_atom_fw ||
2333             !amdgpu_ras_asic_supported(adev))
2334                 return;
2335
2336         if (!adev->gmc.xgmi.connected_to_cpu) {
2337                 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
2338                         dev_info(adev->dev, "MEM ECC is active.\n");
2339                         adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
2340                                                    1 << AMDGPU_RAS_BLOCK__DF);
2341                 } else {
2342                         dev_info(adev->dev, "MEM ECC is not presented.\n");
2343                 }
2344
2345                 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
2346                         dev_info(adev->dev, "SRAM ECC is active.\n");
2347                         if (!amdgpu_sriov_vf(adev)) {
2348                                 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
2349                                                             1 << AMDGPU_RAS_BLOCK__DF);
2350
2351                                 if (adev->ip_versions[VCN_HWIP][0] == IP_VERSION(2, 6, 0) ||
2352                                     adev->ip_versions[VCN_HWIP][0] == IP_VERSION(4, 0, 0))
2353                                         adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
2354                                                         1 << AMDGPU_RAS_BLOCK__JPEG);
2355                                 else
2356                                         adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
2357                                                         1 << AMDGPU_RAS_BLOCK__JPEG);
2358                         } else {
2359                                 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
2360                                                                 1 << AMDGPU_RAS_BLOCK__SDMA |
2361                                                                 1 << AMDGPU_RAS_BLOCK__GFX);
2362                         }
2363                 } else {
2364                         dev_info(adev->dev, "SRAM ECC is not presented.\n");
2365                 }
2366         } else {
2367                 /* driver only manages a few IP blocks RAS feature
2368                  * when GPU is connected cpu through XGMI */
2369                 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
2370                                            1 << AMDGPU_RAS_BLOCK__SDMA |
2371                                            1 << AMDGPU_RAS_BLOCK__MMHUB);
2372         }
2373
2374         amdgpu_ras_get_quirks(adev);
2375
2376         /* hw_supported needs to be aligned with RAS block mask. */
2377         adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
2378
2379         adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
2380                 adev->ras_hw_enabled & amdgpu_ras_mask;
2381 }
2382
2383 static void amdgpu_ras_counte_dw(struct work_struct *work)
2384 {
2385         struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
2386                                               ras_counte_delay_work.work);
2387         struct amdgpu_device *adev = con->adev;
2388         struct drm_device *dev = adev_to_drm(adev);
2389         unsigned long ce_count, ue_count;
2390         int res;
2391
2392         res = pm_runtime_get_sync(dev->dev);
2393         if (res < 0)
2394                 goto Out;
2395
2396         /* Cache new values.
2397          */
2398         if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count) == 0) {
2399                 atomic_set(&con->ras_ce_count, ce_count);
2400                 atomic_set(&con->ras_ue_count, ue_count);
2401         }
2402
2403         pm_runtime_mark_last_busy(dev->dev);
2404 Out:
2405         pm_runtime_put_autosuspend(dev->dev);
2406 }
2407
2408 int amdgpu_ras_init(struct amdgpu_device *adev)
2409 {
2410         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2411         int r;
2412         bool df_poison, umc_poison;
2413
2414         if (con)
2415                 return 0;
2416
2417         con = kmalloc(sizeof(struct amdgpu_ras) +
2418                         sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
2419                         sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
2420                         GFP_KERNEL|__GFP_ZERO);
2421         if (!con)
2422                 return -ENOMEM;
2423
2424         con->adev = adev;
2425         INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
2426         atomic_set(&con->ras_ce_count, 0);
2427         atomic_set(&con->ras_ue_count, 0);
2428
2429         con->objs = (struct ras_manager *)(con + 1);
2430
2431         amdgpu_ras_set_context(adev, con);
2432
2433         amdgpu_ras_check_supported(adev);
2434
2435         if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
2436                 /* set gfx block ras context feature for VEGA20 Gaming
2437                  * send ras disable cmd to ras ta during ras late init.
2438                  */
2439                 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
2440                         con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
2441
2442                         return 0;
2443                 }
2444
2445                 r = 0;
2446                 goto release_con;
2447         }
2448
2449         con->update_channel_flag = false;
2450         con->features = 0;
2451         INIT_LIST_HEAD(&con->head);
2452         /* Might need get this flag from vbios. */
2453         con->flags = RAS_DEFAULT_FLAGS;
2454
2455         /* initialize nbio ras function ahead of any other
2456          * ras functions so hardware fatal error interrupt
2457          * can be enabled as early as possible */
2458         switch (adev->asic_type) {
2459         case CHIP_VEGA20:
2460         case CHIP_ARCTURUS:
2461         case CHIP_ALDEBARAN:
2462                 if (!adev->gmc.xgmi.connected_to_cpu) {
2463                         adev->nbio.ras = &nbio_v7_4_ras;
2464                         amdgpu_ras_register_ras_block(adev, &adev->nbio.ras->ras_block);
2465                         adev->nbio.ras_if = &adev->nbio.ras->ras_block.ras_comm;
2466                 }
2467                 break;
2468         default:
2469                 /* nbio ras is not available */
2470                 break;
2471         }
2472
2473         if (adev->nbio.ras &&
2474             adev->nbio.ras->init_ras_controller_interrupt) {
2475                 r = adev->nbio.ras->init_ras_controller_interrupt(adev);
2476                 if (r)
2477                         goto release_con;
2478         }
2479
2480         if (adev->nbio.ras &&
2481             adev->nbio.ras->init_ras_err_event_athub_interrupt) {
2482                 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
2483                 if (r)
2484                         goto release_con;
2485         }
2486
2487         /* Init poison supported flag, the default value is false */
2488         if (adev->gmc.xgmi.connected_to_cpu) {
2489                 /* enabled by default when GPU is connected to CPU */
2490                 con->poison_supported = true;
2491         }
2492         else if (adev->df.funcs &&
2493             adev->df.funcs->query_ras_poison_mode &&
2494             adev->umc.ras &&
2495             adev->umc.ras->query_ras_poison_mode) {
2496                 df_poison =
2497                         adev->df.funcs->query_ras_poison_mode(adev);
2498                 umc_poison =
2499                         adev->umc.ras->query_ras_poison_mode(adev);
2500                 /* Only poison is set in both DF and UMC, we can support it */
2501                 if (df_poison && umc_poison)
2502                         con->poison_supported = true;
2503                 else if (df_poison != umc_poison)
2504                         dev_warn(adev->dev, "Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
2505                                         df_poison, umc_poison);
2506         }
2507
2508         if (amdgpu_ras_fs_init(adev)) {
2509                 r = -EINVAL;
2510                 goto release_con;
2511         }
2512
2513         dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
2514                  "hardware ability[%x] ras_mask[%x]\n",
2515                  adev->ras_hw_enabled, adev->ras_enabled);
2516
2517         return 0;
2518 release_con:
2519         amdgpu_ras_set_context(adev, NULL);
2520         kfree(con);
2521
2522         return r;
2523 }
2524
2525 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
2526 {
2527         if (adev->gmc.xgmi.connected_to_cpu)
2528                 return 1;
2529         return 0;
2530 }
2531
2532 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
2533                                         struct ras_common_if *ras_block)
2534 {
2535         struct ras_query_if info = {
2536                 .head = *ras_block,
2537         };
2538
2539         if (!amdgpu_persistent_edc_harvesting_supported(adev))
2540                 return 0;
2541
2542         if (amdgpu_ras_query_error_status(adev, &info) != 0)
2543                 DRM_WARN("RAS init harvest failure");
2544
2545         if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
2546                 DRM_WARN("RAS init harvest reset failure");
2547
2548         return 0;
2549 }
2550
2551 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
2552 {
2553        struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2554
2555        if (!con)
2556                return false;
2557
2558        return con->poison_supported;
2559 }
2560
2561 /* helper function to handle common stuff in ip late init phase */
2562 int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
2563                          struct ras_common_if *ras_block)
2564 {
2565         struct amdgpu_ras_block_object *ras_obj = NULL;
2566         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2567         unsigned long ue_count, ce_count;
2568         int r;
2569
2570         /* disable RAS feature per IP block if it is not supported */
2571         if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
2572                 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
2573                 return 0;
2574         }
2575
2576         r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
2577         if (r) {
2578                 if (adev->in_suspend || amdgpu_in_reset(adev)) {
2579                         /* in resume phase, if fail to enable ras,
2580                          * clean up all ras fs nodes, and disable ras */
2581                         goto cleanup;
2582                 } else
2583                         return r;
2584         }
2585
2586         /* check for errors on warm reset edc persisant supported ASIC */
2587         amdgpu_persistent_edc_harvesting(adev, ras_block);
2588
2589         /* in resume phase, no need to create ras fs node */
2590         if (adev->in_suspend || amdgpu_in_reset(adev))
2591                 return 0;
2592
2593         ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
2594         if (ras_obj->ras_cb || (ras_obj->hw_ops &&
2595             (ras_obj->hw_ops->query_poison_status ||
2596             ras_obj->hw_ops->handle_poison_consumption))) {
2597                 r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
2598                 if (r)
2599                         goto cleanup;
2600         }
2601
2602         r = amdgpu_ras_sysfs_create(adev, ras_block);
2603         if (r)
2604                 goto interrupt;
2605
2606         /* Those are the cached values at init.
2607          */
2608         if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count) == 0) {
2609                 atomic_set(&con->ras_ce_count, ce_count);
2610                 atomic_set(&con->ras_ue_count, ue_count);
2611         }
2612
2613         return 0;
2614
2615 interrupt:
2616         if (ras_obj->ras_cb)
2617                 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
2618 cleanup:
2619         amdgpu_ras_feature_enable(adev, ras_block, 0);
2620         return r;
2621 }
2622
2623 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
2624                          struct ras_common_if *ras_block)
2625 {
2626         return amdgpu_ras_block_late_init(adev, ras_block);
2627 }
2628
2629 /* helper function to remove ras fs node and interrupt handler */
2630 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
2631                           struct ras_common_if *ras_block)
2632 {
2633         struct amdgpu_ras_block_object *ras_obj;
2634         if (!ras_block)
2635                 return;
2636
2637         amdgpu_ras_sysfs_remove(adev, ras_block);
2638
2639         ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
2640         if (ras_obj->ras_cb)
2641                 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
2642 }
2643
2644 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
2645                           struct ras_common_if *ras_block)
2646 {
2647         return amdgpu_ras_block_late_fini(adev, ras_block);
2648 }
2649
2650 /* do some init work after IP late init as dependence.
2651  * and it runs in resume/gpu reset/booting up cases.
2652  */
2653 void amdgpu_ras_resume(struct amdgpu_device *adev)
2654 {
2655         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2656         struct ras_manager *obj, *tmp;
2657
2658         if (!adev->ras_enabled || !con) {
2659                 /* clean ras context for VEGA20 Gaming after send ras disable cmd */
2660                 amdgpu_release_ras_context(adev);
2661
2662                 return;
2663         }
2664
2665         if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
2666                 /* Set up all other IPs which are not implemented. There is a
2667                  * tricky thing that IP's actual ras error type should be
2668                  * MULTI_UNCORRECTABLE, but as driver does not handle it, so
2669                  * ERROR_NONE make sense anyway.
2670                  */
2671                 amdgpu_ras_enable_all_features(adev, 1);
2672
2673                 /* We enable ras on all hw_supported block, but as boot
2674                  * parameter might disable some of them and one or more IP has
2675                  * not implemented yet. So we disable them on behalf.
2676                  */
2677                 list_for_each_entry_safe(obj, tmp, &con->head, node) {
2678                         if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
2679                                 amdgpu_ras_feature_enable(adev, &obj->head, 0);
2680                                 /* there should be no any reference. */
2681                                 WARN_ON(alive_obj(obj));
2682                         }
2683                 }
2684         }
2685 }
2686
2687 void amdgpu_ras_suspend(struct amdgpu_device *adev)
2688 {
2689         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2690
2691         if (!adev->ras_enabled || !con)
2692                 return;
2693
2694         amdgpu_ras_disable_all_features(adev, 0);
2695         /* Make sure all ras objects are disabled. */
2696         if (con->features)
2697                 amdgpu_ras_disable_all_features(adev, 1);
2698 }
2699
2700 int amdgpu_ras_late_init(struct amdgpu_device *adev)
2701 {
2702         struct amdgpu_ras_block_list *node, *tmp;
2703         struct amdgpu_ras_block_object *obj;
2704         int r;
2705
2706         /* Guest side doesn't need init ras feature */
2707         if (amdgpu_sriov_vf(adev))
2708                 return 0;
2709
2710         list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
2711                 if (!node->ras_obj) {
2712                         dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
2713                         continue;
2714                 }
2715
2716                 obj = node->ras_obj;
2717                 if (obj->ras_late_init) {
2718                         r = obj->ras_late_init(adev, &obj->ras_comm);
2719                         if (r) {
2720                                 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
2721                                         obj->ras_comm.name, r);
2722                                 return r;
2723                         }
2724                 } else
2725                         amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
2726         }
2727
2728         return 0;
2729 }
2730
2731 /* do some fini work before IP fini as dependence */
2732 int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
2733 {
2734         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2735
2736         if (!adev->ras_enabled || !con)
2737                 return 0;
2738
2739
2740         /* Need disable ras on all IPs here before ip [hw/sw]fini */
2741         if (con->features)
2742                 amdgpu_ras_disable_all_features(adev, 0);
2743         amdgpu_ras_recovery_fini(adev);
2744         return 0;
2745 }
2746
2747 int amdgpu_ras_fini(struct amdgpu_device *adev)
2748 {
2749         struct amdgpu_ras_block_list *ras_node, *tmp;
2750         struct amdgpu_ras_block_object *obj = NULL;
2751         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2752
2753         if (!adev->ras_enabled || !con)
2754                 return 0;
2755
2756         list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
2757                 if (ras_node->ras_obj) {
2758                         obj = ras_node->ras_obj;
2759                         if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
2760                             obj->ras_fini)
2761                                 obj->ras_fini(adev, &obj->ras_comm);
2762                         else
2763                                 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
2764                 }
2765
2766                 /* Clear ras blocks from ras_list and free ras block list node */
2767                 list_del(&ras_node->node);
2768                 kfree(ras_node);
2769         }
2770
2771         amdgpu_ras_fs_fini(adev);
2772         amdgpu_ras_interrupt_remove_all(adev);
2773
2774         WARN(con->features, "Feature mask is not cleared");
2775
2776         if (con->features)
2777                 amdgpu_ras_disable_all_features(adev, 1);
2778
2779         cancel_delayed_work_sync(&con->ras_counte_delay_work);
2780
2781         amdgpu_ras_set_context(adev, NULL);
2782         kfree(con);
2783
2784         return 0;
2785 }
2786
2787 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
2788 {
2789         amdgpu_ras_check_supported(adev);
2790         if (!adev->ras_hw_enabled)
2791                 return;
2792
2793         if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
2794                 dev_info(adev->dev, "uncorrectable hardware error"
2795                         "(ERREVENT_ATHUB_INTERRUPT) detected!\n");
2796
2797                 amdgpu_ras_reset_gpu(adev);
2798         }
2799 }
2800
2801 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
2802 {
2803         if (adev->asic_type == CHIP_VEGA20 &&
2804             adev->pm.fw_version <= 0x283400) {
2805                 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
2806                                 amdgpu_ras_intr_triggered();
2807         }
2808
2809         return false;
2810 }
2811
2812 void amdgpu_release_ras_context(struct amdgpu_device *adev)
2813 {
2814         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2815
2816         if (!con)
2817                 return;
2818
2819         if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
2820                 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
2821                 amdgpu_ras_set_context(adev, NULL);
2822                 kfree(con);
2823         }
2824 }
2825
2826 #ifdef CONFIG_X86_MCE_AMD
2827 static struct amdgpu_device *find_adev(uint32_t node_id)
2828 {
2829         int i;
2830         struct amdgpu_device *adev = NULL;
2831
2832         for (i = 0; i < mce_adev_list.num_gpu; i++) {
2833                 adev = mce_adev_list.devs[i];
2834
2835                 if (adev && adev->gmc.xgmi.connected_to_cpu &&
2836                     adev->gmc.xgmi.physical_node_id == node_id)
2837                         break;
2838                 adev = NULL;
2839         }
2840
2841         return adev;
2842 }
2843
2844 #define GET_MCA_IPID_GPUID(m)   (((m) >> 44) & 0xF)
2845 #define GET_UMC_INST(m)         (((m) >> 21) & 0x7)
2846 #define GET_CHAN_INDEX(m)       ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
2847 #define GPU_ID_OFFSET           8
2848
2849 static int amdgpu_bad_page_notifier(struct notifier_block *nb,
2850                                     unsigned long val, void *data)
2851 {
2852         struct mce *m = (struct mce *)data;
2853         struct amdgpu_device *adev = NULL;
2854         uint32_t gpu_id = 0;
2855         uint32_t umc_inst = 0, ch_inst = 0;
2856
2857         /*
2858          * If the error was generated in UMC_V2, which belongs to GPU UMCs,
2859          * and error occurred in DramECC (Extended error code = 0) then only
2860          * process the error, else bail out.
2861          */
2862         if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
2863                     (XEC(m->status, 0x3f) == 0x0)))
2864                 return NOTIFY_DONE;
2865
2866         /*
2867          * If it is correctable error, return.
2868          */
2869         if (mce_is_correctable(m))
2870                 return NOTIFY_OK;
2871
2872         /*
2873          * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
2874          */
2875         gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
2876
2877         adev = find_adev(gpu_id);
2878         if (!adev) {
2879                 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
2880                                                                 gpu_id);
2881                 return NOTIFY_DONE;
2882         }
2883
2884         /*
2885          * If it is uncorrectable error, then find out UMC instance and
2886          * channel index.
2887          */
2888         umc_inst = GET_UMC_INST(m->ipid);
2889         ch_inst = GET_CHAN_INDEX(m->ipid);
2890
2891         dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
2892                              umc_inst, ch_inst);
2893
2894         if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst))
2895                 return NOTIFY_OK;
2896         else
2897                 return NOTIFY_DONE;
2898 }
2899
2900 static struct notifier_block amdgpu_bad_page_nb = {
2901         .notifier_call  = amdgpu_bad_page_notifier,
2902         .priority       = MCE_PRIO_UC,
2903 };
2904
2905 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
2906 {
2907         /*
2908          * Add the adev to the mce_adev_list.
2909          * During mode2 reset, amdgpu device is temporarily
2910          * removed from the mgpu_info list which can cause
2911          * page retirement to fail.
2912          * Use this list instead of mgpu_info to find the amdgpu
2913          * device on which the UMC error was reported.
2914          */
2915         mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
2916
2917         /*
2918          * Register the x86 notifier only once
2919          * with MCE subsystem.
2920          */
2921         if (notifier_registered == false) {
2922                 mce_register_decode_chain(&amdgpu_bad_page_nb);
2923                 notifier_registered = true;
2924         }
2925 }
2926 #endif
2927
2928 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
2929 {
2930         if (!adev)
2931                 return NULL;
2932
2933         return adev->psp.ras_context.ras;
2934 }
2935
2936 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
2937 {
2938         if (!adev)
2939                 return -EINVAL;
2940
2941         adev->psp.ras_context.ras = ras_con;
2942         return 0;
2943 }
2944
2945 /* check if ras is supported on block, say, sdma, gfx */
2946 int amdgpu_ras_is_supported(struct amdgpu_device *adev,
2947                 unsigned int block)
2948 {
2949         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2950
2951         if (block >= AMDGPU_RAS_BLOCK_COUNT)
2952                 return 0;
2953         return ras && (adev->ras_enabled & (1 << block));
2954 }
2955
2956 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
2957 {
2958         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2959
2960         if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0)
2961                 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
2962         return 0;
2963 }
2964
2965
2966 /* Register each ip ras block into amdgpu ras */
2967 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
2968                 struct amdgpu_ras_block_object *ras_block_obj)
2969 {
2970         struct amdgpu_ras_block_list *ras_node;
2971         if (!adev || !ras_block_obj)
2972                 return -EINVAL;
2973
2974         if (!amdgpu_ras_asic_supported(adev))
2975                 return 0;
2976
2977         ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
2978         if (!ras_node)
2979                 return -ENOMEM;
2980
2981         INIT_LIST_HEAD(&ras_node->node);
2982         ras_node->ras_obj = ras_block_obj;
2983         list_add_tail(&ras_node->node, &adev->ras_list);
2984
2985         return 0;
2986 }