drivers/gpu/drm/amd/amdgpu/umc_v12_0.c

   1 /*
   2  * Copyright 2023 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "umc_v12_0.h"
  24 #include "amdgpu_ras.h"
  25 #include "amdgpu_umc.h"
  26 #include "amdgpu.h"
  27 #include "umc/umc_12_0_0_offset.h"
  28 #include "umc/umc_12_0_0_sh_mask.h"
  29
  30 const uint32_t
  31         umc_v12_0_channel_idx_tbl[]
  32                         [UMC_V12_0_UMC_INSTANCE_NUM]
  33                         [UMC_V12_0_CHANNEL_INSTANCE_NUM] = {
  34                 {{3,   7,   11,  15,  2,   6,   10,  14},  {1,   5,   9,   13,  0,   4,   8,   12},
  35                  {19,  23,  27,  31,  18,  22,  26,  30},  {17,  21,  25,  29,  16,  20,  24,  28}},
  36                 {{47,  43,  39,  35,  46,  42,  38,  34},  {45,  41,  37,  33,  44,  40,  36,  32},
  37                  {63,  59,  55,  51,  62,  58,  54,  50},  {61,  57,  53,  49,  60,  56,  52,  48}},
  38                 {{79,  75,  71,  67,  78,  74,  70,  66},  {77,  73,  69,  65,  76,  72,  68,  64},
  39                  {95,  91,  87,  83,  94,  90,  86,  82},  {93,  89,  85,  81,  92,  88,  84,  80}},
  40                 {{99,  103, 107, 111, 98,  102, 106, 110}, {97,  101, 105, 109, 96,  100, 104, 108},
  41                  {115, 119, 123, 127, 114, 118, 122, 126}, {113, 117, 121, 125, 112, 116, 120, 124}}
  42         };
  43
  44 /* mapping of MCA error address to normalized address */
  45 static const uint32_t umc_v12_0_ma2na_mapping[] = {
  46         0,  5,  6,  8,  9,  14, 12, 13,
  47         10, 11, 15, 16, 17, 18, 19, 20,
  48         21, 22, 23, 24, 25, 26, 27, 28,
  49         24, 7,  29, 30,
  50 };
  51
  52 static inline uint64_t get_umc_v12_0_reg_offset(struct amdgpu_device *adev,
  53                                             uint32_t node_inst,
  54                                             uint32_t umc_inst,
  55                                             uint32_t ch_inst)
  56 {
  57         uint32_t index = umc_inst * adev->umc.channel_inst_num + ch_inst;
  58         uint64_t cross_node_offset = (node_inst == 0) ? 0 : UMC_V12_0_CROSS_NODE_OFFSET;
  59
  60         umc_inst = index / 4;
  61         ch_inst = index % 4;
  62
  63         return adev->umc.channel_offs * ch_inst + UMC_V12_0_INST_DIST * umc_inst +
  64                 UMC_V12_0_NODE_DIST * node_inst + cross_node_offset;
  65 }
  66
  67 static int umc_v12_0_reset_error_count_per_channel(struct amdgpu_device *adev,
  68                                         uint32_t node_inst, uint32_t umc_inst,
  69                                         uint32_t ch_inst, void *data)
  70 {
  71         uint64_t odecc_err_cnt_addr;
  72         uint64_t umc_reg_offset =
  73                 get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
  74
  75         odecc_err_cnt_addr =
  76                 SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccErrCnt);
  77
  78         /* clear error count */
  79         WREG32_PCIE_EXT((odecc_err_cnt_addr + umc_reg_offset) * 4,
  80                         UMC_V12_0_CE_CNT_INIT);
  81
  82         return 0;
  83 }
  84
  85 static void umc_v12_0_reset_error_count(struct amdgpu_device *adev)
  86 {
  87         amdgpu_umc_loop_channels(adev,
  88                 umc_v12_0_reset_error_count_per_channel, NULL);
  89 }
  90
  91 static void umc_v12_0_query_correctable_error_count(struct amdgpu_device *adev,
  92                                                    uint64_t umc_reg_offset,
  93                                                    unsigned long *error_count)
  94 {
  95         uint64_t mc_umc_status;
  96         uint64_t mc_umc_status_addr;
  97
  98         mc_umc_status_addr =
  99                 SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
 100
 101         /* Rely on MCUMC_STATUS for correctable error counter
 102          * MCUMC_STATUS is a 64 bit register
 103          */
 104         mc_umc_status =
 105                 RREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4);
 106
 107         if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
 108             (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1 ||
 109             (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 &&
 110             REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 0)))
 111                 *error_count += 1;
 112 }
 113
 114 static void umc_v12_0_query_uncorrectable_error_count(struct amdgpu_device *adev,
 115                                                       uint64_t umc_reg_offset,
 116                                                       unsigned long *error_count)
 117 {
 118         uint64_t mc_umc_status;
 119         uint64_t mc_umc_status_addr;
 120
 121         mc_umc_status_addr =
 122                 SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
 123
 124         /* Check the MCUMC_STATUS. */
 125         mc_umc_status =
 126                 RREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4);
 127
 128         if ((REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1) &&
 129             (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred) == 1 ||
 130             REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, PCC) == 1 ||
 131             REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 1 ||
 132             REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, TCC) == 1))
 133                 *error_count += 1;
 134 }
 135
 136 static int umc_v12_0_query_error_count(struct amdgpu_device *adev,
 137                                         uint32_t node_inst, uint32_t umc_inst,
 138                                         uint32_t ch_inst, void *data)
 139 {
 140         struct ras_err_data *err_data = (struct ras_err_data *)data;
 141         unsigned long ue_count = 0, ce_count = 0;
 142
 143         /* NOTE: node_inst is converted by adev->umc.active_mask and the range is [0-3],
 144          * which can be used as die ID directly */
 145         struct amdgpu_smuio_mcm_config_info mcm_info = {
 146                 .socket_id = adev->smuio.funcs->get_socket_id(adev),
 147                 .die_id = node_inst,
 148         };
 149
 150         uint64_t umc_reg_offset =
 151                 get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
 152
 153         umc_v12_0_query_correctable_error_count(adev, umc_reg_offset, &ce_count);
 154         umc_v12_0_query_uncorrectable_error_count(adev, umc_reg_offset, &ue_count);
 155
 156         amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count);
 157         amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count);
 158
 159         return 0;
 160 }
 161
 162 static void umc_v12_0_query_ras_error_count(struct amdgpu_device *adev,
 163                                            void *ras_error_status)
 164 {
 165         amdgpu_umc_loop_channels(adev,
 166                 umc_v12_0_query_error_count, ras_error_status);
 167
 168         umc_v12_0_reset_error_count(adev);
 169 }
 170
 171 static bool umc_v12_0_bit_wise_xor(uint32_t val)
 172 {
 173         bool result = 0;
 174         int i;
 175
 176         for (i = 0; i < 32; i++)
 177                 result = result ^ ((val >> i) & 0x1);
 178
 179         return result;
 180 }
 181
 182 static void umc_v12_0_convert_error_address(struct amdgpu_device *adev,
 183                                             struct ras_err_data *err_data, uint64_t err_addr,
 184                                             uint32_t ch_inst, uint32_t umc_inst,
 185                                             uint32_t node_inst)
 186 {
 187         uint32_t channel_index, i;
 188         uint64_t soc_pa, na, retired_page, column;
 189         uint32_t bank_hash0, bank_hash1, bank_hash2, bank_hash3, col, row, row_xor;
 190         uint32_t bank0, bank1, bank2, bank3, bank;
 191
 192         bank_hash0 = (err_addr >> UMC_V12_0_MCA_B0_BIT) & 0x1ULL;
 193         bank_hash1 = (err_addr >> UMC_V12_0_MCA_B1_BIT) & 0x1ULL;
 194         bank_hash2 = (err_addr >> UMC_V12_0_MCA_B2_BIT) & 0x1ULL;
 195         bank_hash3 = (err_addr >> UMC_V12_0_MCA_B3_BIT) & 0x1ULL;
 196         col = (err_addr >> 1) & 0x1fULL;
 197         row = (err_addr >> 10) & 0x3fffULL;
 198
 199         /* apply bank hash algorithm */
 200         bank0 =
 201                 bank_hash0 ^ (UMC_V12_0_XOR_EN0 &
 202                 (umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR0) ^
 203                 (umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR0))));
 204         bank1 =
 205                 bank_hash1 ^ (UMC_V12_0_XOR_EN1 &
 206                 (umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR1) ^
 207                 (umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR1))));
 208         bank2 =
 209                 bank_hash2 ^ (UMC_V12_0_XOR_EN2 &
 210                 (umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR2) ^
 211                 (umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR2))));
 212         bank3 =
 213                 bank_hash3 ^ (UMC_V12_0_XOR_EN3 &
 214                 (umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR3) ^
 215                 (umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR3))));
 216
 217         bank = bank0 | (bank1 << 1) | (bank2 << 2) | (bank3 << 3);
 218         err_addr &= ~0x3c0ULL;
 219         err_addr |= (bank << UMC_V12_0_MCA_B0_BIT);
 220
 221         na = 0x0;
 222         /* convert mca error address to normalized address */
 223         for (i = 1; i < ARRAY_SIZE(umc_v12_0_ma2na_mapping); i++)
 224                 na |= ((err_addr >> i) & 0x1ULL) << umc_v12_0_ma2na_mapping[i];
 225
 226         channel_index =
 227                 adev->umc.channel_idx_tbl[node_inst * adev->umc.umc_inst_num *
 228                         adev->umc.channel_inst_num +
 229                         umc_inst * adev->umc.channel_inst_num +
 230                         ch_inst];
 231         /* translate umc channel address to soc pa, 3 parts are included */
 232         soc_pa = ADDR_OF_32KB_BLOCK(na) |
 233                 ADDR_OF_256B_BLOCK(channel_index) |
 234                 OFFSET_IN_256B_BLOCK(na);
 235
 236         /* the umc channel bits are not original values, they are hashed */
 237         UMC_V12_0_SET_CHANNEL_HASH(channel_index, soc_pa);
 238
 239         /* clear [C3 C2] in soc physical address */
 240         soc_pa &= ~(0x3ULL << UMC_V12_0_PA_C2_BIT);
 241         /* clear [C4] in soc physical address */
 242         soc_pa &= ~(0x1ULL << UMC_V12_0_PA_C4_BIT);
 243
 244         row_xor = row ^ (0x1ULL << 13);
 245         /* loop for all possibilities of [C4 C3 C2] */
 246         for (column = 0; column < UMC_V12_0_NA_MAP_PA_NUM; column++) {
 247                 retired_page = soc_pa | ((column & 0x3) << UMC_V12_0_PA_C2_BIT);
 248                 retired_page |= (((column & 0x4) >> 2) << UMC_V12_0_PA_C4_BIT);
 249                 /* include column bit 0 and 1 */
 250                 col &= 0x3;
 251                 col |= (column << 2);
 252                 dev_info(adev->dev,
 253                         "Error Address(PA):0x%-10llx Row:0x%-4x Col:0x%-2x Bank:0x%x Channel:0x%x\n",
 254                         retired_page, row, col, bank, channel_index);
 255                 amdgpu_umc_fill_error_record(err_data, err_addr,
 256                         retired_page, channel_index, umc_inst);
 257
 258                 /* shift R13 bit */
 259                 retired_page ^= (0x1ULL << UMC_V12_0_PA_R13_BIT);
 260                 dev_info(adev->dev,
 261                         "Error Address(PA):0x%-10llx Row:0x%-4x Col:0x%-2x Bank:0x%x Channel:0x%x\n",
 262                         retired_page, row_xor, col, bank, channel_index);
 263                 amdgpu_umc_fill_error_record(err_data, err_addr,
 264                         retired_page, channel_index, umc_inst);
 265         }
 266 }
 267
 268 static int umc_v12_0_query_error_address(struct amdgpu_device *adev,
 269                                         uint32_t node_inst, uint32_t umc_inst,
 270                                         uint32_t ch_inst, void *data)
 271 {
 272         uint64_t mc_umc_status_addr;
 273         uint64_t mc_umc_status, err_addr;
 274         uint64_t mc_umc_addrt0;
 275         struct ras_err_data *err_data = (struct ras_err_data *)data;
 276         uint64_t umc_reg_offset =
 277                 get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
 278
 279         mc_umc_status_addr =
 280                 SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
 281
 282         mc_umc_status = RREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4);
 283
 284         if (mc_umc_status == 0)
 285                 return 0;
 286
 287         if (!err_data->err_addr) {
 288                 /* clear umc status */
 289                 WREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
 290
 291                 return 0;
 292         }
 293
 294         /* calculate error address if ue error is detected */
 295         if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
 296             REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, AddrV) == 1 &&
 297             REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 1) {
 298
 299                 mc_umc_addrt0 =
 300                         SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_ADDRT0);
 301
 302                 err_addr = RREG64_PCIE_EXT((mc_umc_addrt0 + umc_reg_offset) * 4);
 303
 304                 err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
 305
 306                 umc_v12_0_convert_error_address(adev, err_data, err_addr,
 307                                         ch_inst, umc_inst, node_inst);
 308         }
 309
 310         /* clear umc status */
 311         WREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
 312
 313         return 0;
 314 }
 315
 316 static void umc_v12_0_query_ras_error_address(struct amdgpu_device *adev,
 317                                              void *ras_error_status)
 318 {
 319         amdgpu_umc_loop_channels(adev,
 320                 umc_v12_0_query_error_address, ras_error_status);
 321 }
 322
 323 static int umc_v12_0_err_cnt_init_per_channel(struct amdgpu_device *adev,
 324                                         uint32_t node_inst, uint32_t umc_inst,
 325                                         uint32_t ch_inst, void *data)
 326 {
 327         uint32_t odecc_cnt_sel;
 328         uint64_t odecc_cnt_sel_addr, odecc_err_cnt_addr;
 329         uint64_t umc_reg_offset =
 330                 get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
 331
 332         odecc_cnt_sel_addr =
 333                 SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccCntSel);
 334         odecc_err_cnt_addr =
 335                 SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccErrCnt);
 336
 337         odecc_cnt_sel = RREG32_PCIE_EXT((odecc_cnt_sel_addr + umc_reg_offset) * 4);
 338
 339         /* set ce error interrupt type to APIC based interrupt */
 340         odecc_cnt_sel = REG_SET_FIELD(odecc_cnt_sel, UMCCH0_OdEccCntSel,
 341                                         OdEccErrInt, 0x1);
 342         WREG32_PCIE_EXT((odecc_cnt_sel_addr + umc_reg_offset) * 4, odecc_cnt_sel);
 343
 344         /* set error count to initial value */
 345         WREG32_PCIE_EXT((odecc_err_cnt_addr + umc_reg_offset) * 4, UMC_V12_0_CE_CNT_INIT);
 346
 347         return 0;
 348 }
 349
 350 static void umc_v12_0_err_cnt_init(struct amdgpu_device *adev)
 351 {
 352         amdgpu_umc_loop_channels(adev,
 353                 umc_v12_0_err_cnt_init_per_channel, NULL);
 354 }
 355
 356 static bool umc_v12_0_query_ras_poison_mode(struct amdgpu_device *adev)
 357 {
 358         /*
 359          * Force return true, because regUMCCH0_EccCtrl
 360          * is not accessible from host side
 361          */
 362         return true;
 363 }
 364
 365 const struct amdgpu_ras_block_hw_ops umc_v12_0_ras_hw_ops = {
 366         .query_ras_error_count = umc_v12_0_query_ras_error_count,
 367         .query_ras_error_address = umc_v12_0_query_ras_error_address,
 368 };
 369
 370 struct amdgpu_umc_ras umc_v12_0_ras = {
 371         .ras_block = {
 372                 .hw_ops = &umc_v12_0_ras_hw_ops,
 373         },
 374         .err_cnt_init = umc_v12_0_err_cnt_init,
 375         .query_ras_poison_mode = umc_v12_0_query_ras_poison_mode,
 376 };