drivers/gpu/drm/amd/amdgpu/umc_v12_0.c

   1 /*
   2  * Copyright 2023 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "umc_v12_0.h"
  24 #include "amdgpu_ras.h"
  25 #include "amdgpu_umc.h"
  26 #include "amdgpu.h"
  27 #include "umc/umc_12_0_0_offset.h"
  28 #include "umc/umc_12_0_0_sh_mask.h"
  29
  30 const uint32_t
  31         umc_v12_0_channel_idx_tbl[]
  32                         [UMC_V12_0_UMC_INSTANCE_NUM]
  33                         [UMC_V12_0_CHANNEL_INSTANCE_NUM] = {
  34                 {{3,   7,   11,  15,  2,   6,   10,  14},  {1,   5,   9,   13,  0,   4,   8,   12},
  35                  {19,  23,  27,  31,  18,  22,  26,  30},  {17,  21,  25,  29,  16,  20,  24,  28}},
  36                 {{47,  43,  39,  35,  46,  42,  38,  34},  {45,  41,  37,  33,  44,  40,  36,  32},
  37                  {63,  59,  55,  51,  62,  58,  54,  50},  {61,  57,  53,  49,  60,  56,  52,  48}},
  38                 {{79,  75,  71,  67,  78,  74,  70,  66},  {77,  73,  69,  65,  76,  72,  68,  64},
  39                  {95,  91,  87,  83,  94,  90,  86,  82},  {93,  89,  85,  81,  92,  88,  84,  80}},
  40                 {{99,  103, 107, 111, 98,  102, 106, 110}, {97,  101, 105, 109, 96,  100, 104, 108},
  41                  {115, 119, 123, 127, 114, 118, 122, 126}, {113, 117, 121, 125, 112, 116, 120, 124}}
  42         };
  43
  44 /* mapping of MCA error address to normalized address */
  45 static const uint32_t umc_v12_0_ma2na_mapping[] = {
  46         0,  5,  6,  8,  9,  14, 12, 13,
  47         10, 11, 15, 16, 17, 18, 19, 20,
  48         21, 22, 23, 24, 25, 26, 27, 28,
  49         24, 7,  29, 30,
  50 };
  51
  52 static inline uint64_t get_umc_v12_0_reg_offset(struct amdgpu_device *adev,
  53                                             uint32_t node_inst,
  54                                             uint32_t umc_inst,
  55                                             uint32_t ch_inst)
  56 {
  57         uint32_t index = umc_inst * adev->umc.channel_inst_num + ch_inst;
  58         uint64_t cross_node_offset = (node_inst == 0) ? 0 : UMC_V12_0_CROSS_NODE_OFFSET;
  59
  60         umc_inst = index / 4;
  61         ch_inst = index % 4;
  62
  63         return adev->umc.channel_offs * ch_inst + UMC_V12_0_INST_DIST * umc_inst +
  64                 UMC_V12_0_NODE_DIST * node_inst + cross_node_offset;
  65 }
  66
  67 static int umc_v12_0_reset_error_count_per_channel(struct amdgpu_device *adev,
  68                                         uint32_t node_inst, uint32_t umc_inst,
  69                                         uint32_t ch_inst, void *data)
  70 {
  71         uint64_t odecc_err_cnt_addr;
  72         uint64_t umc_reg_offset =
  73                 get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
  74
  75         odecc_err_cnt_addr =
  76                 SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccErrCnt);
  77
  78         /* clear error count */
  79         WREG32_PCIE_EXT((odecc_err_cnt_addr + umc_reg_offset) * 4,
  80                         UMC_V12_0_CE_CNT_INIT);
  81
  82         return 0;
  83 }
  84
  85 static void umc_v12_0_reset_error_count(struct amdgpu_device *adev)
  86 {
  87         amdgpu_umc_loop_channels(adev,
  88                 umc_v12_0_reset_error_count_per_channel, NULL);
  89 }
  90
  91 static bool umc_v12_0_is_uncorrectable_error(uint64_t mc_umc_status)
  92 {
  93         return ((REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1) &&
  94                 (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred) == 1 ||
  95                 REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, PCC) == 1 ||
  96                 REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 1 ||
  97                 REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, TCC) == 1));
  98 }
  99
 100 static bool umc_v12_0_is_correctable_error(uint64_t mc_umc_status)
 101 {
 102         return (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
 103                 (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1 ||
 104                 (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 &&
 105                 REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 0) ||
 106                 /* Identify data parity error in replay mode */
 107                 ((REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, ErrorCodeExt) == 0x5 ||
 108                 REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, ErrorCodeExt) == 0xb) &&
 109                 !(umc_v12_0_is_uncorrectable_error(mc_umc_status)))));
 110 }
 111
 112 static void umc_v12_0_query_correctable_error_count(struct amdgpu_device *adev,
 113                                                    uint64_t umc_reg_offset,
 114                                                    unsigned long *error_count)
 115 {
 116         uint64_t mc_umc_status;
 117         uint64_t mc_umc_status_addr;
 118
 119         mc_umc_status_addr =
 120                 SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
 121
 122         /* Rely on MCUMC_STATUS for correctable error counter
 123          * MCUMC_STATUS is a 64 bit register
 124          */
 125         mc_umc_status =
 126                 RREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4);
 127
 128         if (umc_v12_0_is_correctable_error(mc_umc_status))
 129                 *error_count += 1;
 130 }
 131
 132 static void umc_v12_0_query_uncorrectable_error_count(struct amdgpu_device *adev,
 133                                                       uint64_t umc_reg_offset,
 134                                                       unsigned long *error_count)
 135 {
 136         uint64_t mc_umc_status;
 137         uint64_t mc_umc_status_addr;
 138
 139         mc_umc_status_addr =
 140                 SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
 141
 142         /* Check the MCUMC_STATUS. */
 143         mc_umc_status =
 144                 RREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4);
 145
 146         if (umc_v12_0_is_uncorrectable_error(mc_umc_status))
 147                 *error_count += 1;
 148 }
 149
 150 static int umc_v12_0_query_error_count(struct amdgpu_device *adev,
 151                                         uint32_t node_inst, uint32_t umc_inst,
 152                                         uint32_t ch_inst, void *data)
 153 {
 154         struct ras_err_data *err_data = (struct ras_err_data *)data;
 155         unsigned long ue_count = 0, ce_count = 0;
 156
 157         /* NOTE: node_inst is converted by adev->umc.active_mask and the range is [0-3],
 158          * which can be used as die ID directly */
 159         struct amdgpu_smuio_mcm_config_info mcm_info = {
 160                 .socket_id = adev->smuio.funcs->get_socket_id(adev),
 161                 .die_id = node_inst,
 162         };
 163
 164         uint64_t umc_reg_offset =
 165                 get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
 166
 167         umc_v12_0_query_correctable_error_count(adev, umc_reg_offset, &ce_count);
 168         umc_v12_0_query_uncorrectable_error_count(adev, umc_reg_offset, &ue_count);
 169
 170         amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count);
 171         amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count);
 172
 173         return 0;
 174 }
 175
 176 static void umc_v12_0_query_ras_error_count(struct amdgpu_device *adev,
 177                                            void *ras_error_status)
 178 {
 179         amdgpu_umc_loop_channels(adev,
 180                 umc_v12_0_query_error_count, ras_error_status);
 181
 182         umc_v12_0_reset_error_count(adev);
 183 }
 184
 185 static bool umc_v12_0_bit_wise_xor(uint32_t val)
 186 {
 187         bool result = 0;
 188         int i;
 189
 190         for (i = 0; i < 32; i++)
 191                 result = result ^ ((val >> i) & 0x1);
 192
 193         return result;
 194 }
 195
 196 static void umc_v12_0_convert_error_address(struct amdgpu_device *adev,
 197                                             struct ras_err_data *err_data, uint64_t err_addr,
 198                                             uint32_t ch_inst, uint32_t umc_inst,
 199                                             uint32_t node_inst)
 200 {
 201         uint32_t channel_index, i;
 202         uint64_t soc_pa, na, retired_page, column;
 203         uint32_t bank_hash0, bank_hash1, bank_hash2, bank_hash3, col, row, row_xor;
 204         uint32_t bank0, bank1, bank2, bank3, bank;
 205
 206         bank_hash0 = (err_addr >> UMC_V12_0_MCA_B0_BIT) & 0x1ULL;
 207         bank_hash1 = (err_addr >> UMC_V12_0_MCA_B1_BIT) & 0x1ULL;
 208         bank_hash2 = (err_addr >> UMC_V12_0_MCA_B2_BIT) & 0x1ULL;
 209         bank_hash3 = (err_addr >> UMC_V12_0_MCA_B3_BIT) & 0x1ULL;
 210         col = (err_addr >> 1) & 0x1fULL;
 211         row = (err_addr >> 10) & 0x3fffULL;
 212
 213         /* apply bank hash algorithm */
 214         bank0 =
 215                 bank_hash0 ^ (UMC_V12_0_XOR_EN0 &
 216                 (umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR0) ^
 217                 (umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR0))));
 218         bank1 =
 219                 bank_hash1 ^ (UMC_V12_0_XOR_EN1 &
 220                 (umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR1) ^
 221                 (umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR1))));
 222         bank2 =
 223                 bank_hash2 ^ (UMC_V12_0_XOR_EN2 &
 224                 (umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR2) ^
 225                 (umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR2))));
 226         bank3 =
 227                 bank_hash3 ^ (UMC_V12_0_XOR_EN3 &
 228                 (umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR3) ^
 229                 (umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR3))));
 230
 231         bank = bank0 | (bank1 << 1) | (bank2 << 2) | (bank3 << 3);
 232         err_addr &= ~0x3c0ULL;
 233         err_addr |= (bank << UMC_V12_0_MCA_B0_BIT);
 234
 235         na = 0x0;
 236         /* convert mca error address to normalized address */
 237         for (i = 1; i < ARRAY_SIZE(umc_v12_0_ma2na_mapping); i++)
 238                 na |= ((err_addr >> i) & 0x1ULL) << umc_v12_0_ma2na_mapping[i];
 239
 240         channel_index =
 241                 adev->umc.channel_idx_tbl[node_inst * adev->umc.umc_inst_num *
 242                         adev->umc.channel_inst_num +
 243                         umc_inst * adev->umc.channel_inst_num +
 244                         ch_inst];
 245         /* translate umc channel address to soc pa, 3 parts are included */
 246         soc_pa = ADDR_OF_32KB_BLOCK(na) |
 247                 ADDR_OF_256B_BLOCK(channel_index) |
 248                 OFFSET_IN_256B_BLOCK(na);
 249
 250         /* the umc channel bits are not original values, they are hashed */
 251         UMC_V12_0_SET_CHANNEL_HASH(channel_index, soc_pa);
 252
 253         /* clear [C3 C2] in soc physical address */
 254         soc_pa &= ~(0x3ULL << UMC_V12_0_PA_C2_BIT);
 255         /* clear [C4] in soc physical address */
 256         soc_pa &= ~(0x1ULL << UMC_V12_0_PA_C4_BIT);
 257
 258         row_xor = row ^ (0x1ULL << 13);
 259         /* loop for all possibilities of [C4 C3 C2] */
 260         for (column = 0; column < UMC_V12_0_NA_MAP_PA_NUM; column++) {
 261                 retired_page = soc_pa | ((column & 0x3) << UMC_V12_0_PA_C2_BIT);
 262                 retired_page |= (((column & 0x4) >> 2) << UMC_V12_0_PA_C4_BIT);
 263                 /* include column bit 0 and 1 */
 264                 col &= 0x3;
 265                 col |= (column << 2);
 266                 dev_info(adev->dev,
 267                         "Error Address(PA):0x%-10llx Row:0x%-4x Col:0x%-2x Bank:0x%x Channel:0x%x\n",
 268                         retired_page, row, col, bank, channel_index);
 269                 amdgpu_umc_fill_error_record(err_data, err_addr,
 270                         retired_page, channel_index, umc_inst);
 271
 272                 /* shift R13 bit */
 273                 retired_page ^= (0x1ULL << UMC_V12_0_PA_R13_BIT);
 274                 dev_info(adev->dev,
 275                         "Error Address(PA):0x%-10llx Row:0x%-4x Col:0x%-2x Bank:0x%x Channel:0x%x\n",
 276                         retired_page, row_xor, col, bank, channel_index);
 277                 amdgpu_umc_fill_error_record(err_data, err_addr,
 278                         retired_page, channel_index, umc_inst);
 279         }
 280 }
 281
 282 static int umc_v12_0_query_error_address(struct amdgpu_device *adev,
 283                                         uint32_t node_inst, uint32_t umc_inst,
 284                                         uint32_t ch_inst, void *data)
 285 {
 286         uint64_t mc_umc_status_addr;
 287         uint64_t mc_umc_status, err_addr;
 288         uint64_t mc_umc_addrt0;
 289         struct ras_err_data *err_data = (struct ras_err_data *)data;
 290         uint64_t umc_reg_offset =
 291                 get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
 292
 293         mc_umc_status_addr =
 294                 SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
 295
 296         mc_umc_status = RREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4);
 297
 298         if (mc_umc_status == 0)
 299                 return 0;
 300
 301         if (!err_data->err_addr) {
 302                 /* clear umc status */
 303                 WREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
 304
 305                 return 0;
 306         }
 307
 308         /* calculate error address if ue error is detected */
 309         if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
 310             REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, AddrV) == 1 &&
 311             REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 1) {
 312
 313                 mc_umc_addrt0 =
 314                         SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_ADDRT0);
 315
 316                 err_addr = RREG64_PCIE_EXT((mc_umc_addrt0 + umc_reg_offset) * 4);
 317
 318                 err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
 319
 320                 umc_v12_0_convert_error_address(adev, err_data, err_addr,
 321                                         ch_inst, umc_inst, node_inst);
 322         }
 323
 324         /* clear umc status */
 325         WREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
 326
 327         return 0;
 328 }
 329
 330 static void umc_v12_0_query_ras_error_address(struct amdgpu_device *adev,
 331                                              void *ras_error_status)
 332 {
 333         amdgpu_umc_loop_channels(adev,
 334                 umc_v12_0_query_error_address, ras_error_status);
 335 }
 336
 337 static int umc_v12_0_err_cnt_init_per_channel(struct amdgpu_device *adev,
 338                                         uint32_t node_inst, uint32_t umc_inst,
 339                                         uint32_t ch_inst, void *data)
 340 {
 341         uint32_t odecc_cnt_sel;
 342         uint64_t odecc_cnt_sel_addr, odecc_err_cnt_addr;
 343         uint64_t umc_reg_offset =
 344                 get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
 345
 346         odecc_cnt_sel_addr =
 347                 SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccCntSel);
 348         odecc_err_cnt_addr =
 349                 SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccErrCnt);
 350
 351         odecc_cnt_sel = RREG32_PCIE_EXT((odecc_cnt_sel_addr + umc_reg_offset) * 4);
 352
 353         /* set ce error interrupt type to APIC based interrupt */
 354         odecc_cnt_sel = REG_SET_FIELD(odecc_cnt_sel, UMCCH0_OdEccCntSel,
 355                                         OdEccErrInt, 0x1);
 356         WREG32_PCIE_EXT((odecc_cnt_sel_addr + umc_reg_offset) * 4, odecc_cnt_sel);
 357
 358         /* set error count to initial value */
 359         WREG32_PCIE_EXT((odecc_err_cnt_addr + umc_reg_offset) * 4, UMC_V12_0_CE_CNT_INIT);
 360
 361         return 0;
 362 }
 363
 364 static void umc_v12_0_err_cnt_init(struct amdgpu_device *adev)
 365 {
 366         amdgpu_umc_loop_channels(adev,
 367                 umc_v12_0_err_cnt_init_per_channel, NULL);
 368 }
 369
 370 static bool umc_v12_0_query_ras_poison_mode(struct amdgpu_device *adev)
 371 {
 372         /*
 373          * Force return true, because regUMCCH0_EccCtrl
 374          * is not accessible from host side
 375          */
 376         return true;
 377 }
 378
 379 const struct amdgpu_ras_block_hw_ops umc_v12_0_ras_hw_ops = {
 380         .query_ras_error_count = umc_v12_0_query_ras_error_count,
 381         .query_ras_error_address = umc_v12_0_query_ras_error_address,
 382 };
 383
 384 struct amdgpu_umc_ras umc_v12_0_ras = {
 385         .ras_block = {
 386                 .hw_ops = &umc_v12_0_ras_hw_ops,
 387         },
 388         .err_cnt_init = umc_v12_0_err_cnt_init,
 389         .query_ras_poison_mode = umc_v12_0_query_ras_poison_mode,
 390 };