drivers/gpu/drm/amd/amdgpu/df_v3_6.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "amdgpu.h"
  24 #include "df_v3_6.h"
  25
  26 #include "df/df_3_6_default.h"
  27 #include "df/df_3_6_offset.h"
  28 #include "df/df_3_6_sh_mask.h"
  29
  30 static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
  31                                        16, 32, 0, 0, 0, 2, 4, 8};
  32
  33 static void df_v3_6_init(struct amdgpu_device *adev)
  34 {
  35 }
  36
  37 static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
  38                                           bool enable)
  39 {
  40         u32 tmp;
  41
  42         if (enable) {
  43                 tmp = RREG32_SOC15(DF, 0, mmFabricConfigAccessControl);
  44                 tmp &= ~FabricConfigAccessControl__CfgRegInstAccEn_MASK;
  45                 WREG32_SOC15(DF, 0, mmFabricConfigAccessControl, tmp);
  46         } else
  47                 WREG32_SOC15(DF, 0, mmFabricConfigAccessControl,
  48                              mmFabricConfigAccessControl_DEFAULT);
  49 }
  50
  51 static u32 df_v3_6_get_fb_channel_number(struct amdgpu_device *adev)
  52 {
  53         u32 tmp;
  54
  55         tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DramBaseAddress0);
  56         tmp &= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;
  57         tmp >>= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
  58
  59         return tmp;
  60 }
  61
  62 static u32 df_v3_6_get_hbm_channel_number(struct amdgpu_device *adev)
  63 {
  64         int fb_channel_number;
  65
  66         fb_channel_number = adev->df_funcs->get_fb_channel_number(adev);
  67         if (fb_channel_number >= ARRAY_SIZE(df_v3_6_channel_number))
  68                 fb_channel_number = 0;
  69
  70         return df_v3_6_channel_number[fb_channel_number];
  71 }
  72
  73 static void df_v3_6_update_medium_grain_clock_gating(struct amdgpu_device *adev,
  74                                                      bool enable)
  75 {
  76         u32 tmp;
  77
  78         /* Put DF on broadcast mode */
  79         adev->df_funcs->enable_broadcast_mode(adev, true);
  80
  81         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DF_MGCG)) {
  82                 tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
  83                 tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
  84                 tmp |= DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY;
  85                 WREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater, tmp);
  86         } else {
  87                 tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
  88                 tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
  89                 tmp |= DF_V3_6_MGCG_DISABLE;
  90                 WREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater, tmp);
  91         }
  92
  93         /* Exit broadcast mode */
  94         adev->df_funcs->enable_broadcast_mode(adev, false);
  95 }
  96
  97 static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
  98                                           u32 *flags)
  99 {
 100         u32 tmp;
 101
 102         /* AMD_CG_SUPPORT_DF_MGCG */
 103         tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
 104         if (tmp & DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY)
 105                 *flags |= AMD_CG_SUPPORT_DF_MGCG;
 106 }
 107
 108 /* hold counter assignment per gpu struct */
 109 struct df_v3_6_event_mask {
 110                 struct amdgpu_device gpu;
 111                 uint64_t config_assign_mask[AMDGPU_DF_MAX_COUNTERS];
 112 };
 113
 114 /* get assigned df perfmon ctr as int */
 115 static void df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
 116                                       uint64_t config,
 117                                       int *counter)
 118 {
 119         struct df_v3_6_event_mask *mask;
 120         int i;
 121
 122         mask = container_of(adev, struct df_v3_6_event_mask, gpu);
 123
 124         for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
 125                 if ((config & 0x0FFFFFFUL) == mask->config_assign_mask[i]) {
 126                         *counter = i;
 127                         return;
 128                 }
 129         }
 130 }
 131
 132 /* get address based on counter assignment */
 133 static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
 134                                  uint64_t config,
 135                                  int is_ctrl,
 136                                  uint32_t *lo_base_addr,
 137                                  uint32_t *hi_base_addr)
 138 {
 139
 140         int target_cntr = -1;
 141
 142         df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
 143
 144         if (target_cntr < 0)
 145                 return;
 146
 147         switch (target_cntr) {
 148
 149         case 0:
 150                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo0 : smnPerfMonCtrLo0;
 151                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi0 : smnPerfMonCtrHi0;
 152                 break;
 153         case 1:
 154                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo1 : smnPerfMonCtrLo1;
 155                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi1 : smnPerfMonCtrHi1;
 156                 break;
 157         case 2:
 158                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo2 : smnPerfMonCtrLo2;
 159                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi2 : smnPerfMonCtrHi2;
 160                 break;
 161         case 3:
 162                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo3 : smnPerfMonCtrLo3;
 163                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi3 : smnPerfMonCtrHi3;
 164                 break;
 165
 166         }
 167
 168 }
 169
 170 /* get read counter address */
 171 static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
 172                                           uint64_t config,
 173                                           uint32_t *lo_base_addr,
 174                                           uint32_t *hi_base_addr)
 175 {
 176         df_v3_6_pmc_get_addr(adev, config, 0, lo_base_addr, hi_base_addr);
 177 }
 178
 179 /* get control counter settings i.e. address and values to set */
 180 static void df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
 181                                           uint64_t config,
 182                                           uint32_t *lo_base_addr,
 183                                           uint32_t *hi_base_addr,
 184                                           uint32_t *lo_val,
 185                                           uint32_t *hi_val)
 186 {
 187
 188         uint32_t eventsel, instance, unitmask;
 189         uint32_t es_5_0, es_13_0, es_13_6, es_13_12, es_11_8, es_7_0;
 190
 191         df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
 192
 193         if (lo_val == NULL || hi_val == NULL)
 194                 return;
 195
 196         if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
 197                 DRM_ERROR("DF PMC addressing not retrieved! Lo: %x, Hi: %x",
 198                                 *lo_base_addr, *hi_base_addr);
 199                 return;
 200         }
 201
 202         eventsel = GET_EVENT(config);
 203         instance = GET_INSTANCE(config);
 204         unitmask = GET_UNITMASK(config);
 205
 206         es_5_0 = eventsel & 0x3FUL;
 207         es_13_6 = instance;
 208         es_13_0 = (es_13_6 << 6) + es_5_0;
 209         es_13_12 = (es_13_0 & 0x03000UL) >> 12;
 210         es_11_8 = (es_13_0 & 0x0F00UL) >> 8;
 211         es_7_0 = es_13_0 & 0x0FFUL;
 212         *lo_val = (es_7_0 & 0xFFUL) | ((unitmask & 0x0FUL) << 8);
 213         *hi_val = (es_11_8 | ((es_13_12)<<(29)));
 214 }
 215
 216 /* assign df performance counters for read */
 217 static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
 218                                    uint64_t config,
 219                                    int *is_assigned)
 220 {
 221
 222         struct df_v3_6_event_mask *mask;
 223         int i, target_cntr;
 224
 225         target_cntr = -1;
 226
 227         *is_assigned = 0;
 228
 229         df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
 230
 231         if (target_cntr >= 0) {
 232                 *is_assigned = 1;
 233                 return 0;
 234         }
 235
 236         mask = container_of(adev, struct df_v3_6_event_mask, gpu);
 237
 238         for (i = 0; i < AMDGPU_DF_MAX_COUNTERS; i++) {
 239                 if (mask->config_assign_mask[i] == 0ULL) {
 240                         mask->config_assign_mask[i] = config & 0x0FFFFFFUL;
 241                         return 0;
 242                 }
 243         }
 244
 245         return -ENOSPC;
 246 }
 247
 248 /* release performance counter */
 249 static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
 250                                      uint64_t config)
 251 {
 252
 253         struct df_v3_6_event_mask *mask;
 254         int target_cntr;
 255
 256         target_cntr = -1;
 257
 258         df_v3_6_pmc_config_2_cntr(adev, config, &target_cntr);
 259
 260         mask = container_of(adev, struct df_v3_6_event_mask, gpu);
 261
 262         if (target_cntr >= 0)
 263                 mask->config_assign_mask[target_cntr] = 0ULL;
 264
 265 }
 266
 267 /*
 268  * get xgmi link counters via programmable data fabric (df) counters (max 4)
 269  * using cake tx event.
 270  *
 271  * @adev -> amdgpu device
 272  * @instance-> currently cake has 2 links to poll on vega20
 273  * @count -> counters to pass
 274  *
 275  */
 276
 277 static void df_v3_6_get_xgmi_link_cntr(struct amdgpu_device *adev,
 278                                        int instance,
 279                                        uint64_t *count)
 280 {
 281         uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
 282         uint64_t config;
 283
 284         config = GET_INSTANCE_CONFIG(instance);
 285
 286         df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
 287                                       &hi_base_addr);
 288
 289         if ((lo_base_addr == 0) || (hi_base_addr == 0))
 290                 return;
 291
 292         lo_val = RREG32_PCIE(lo_base_addr);
 293         hi_val = RREG32_PCIE(hi_base_addr);
 294
 295         *count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
 296 }
 297
 298 /*
 299  * reset xgmi link counters
 300  *
 301  * @adev -> amdgpu device
 302  * @instance-> currently cake has 2 links to poll on vega20
 303  *
 304  */
 305 static void df_v3_6_reset_xgmi_link_cntr(struct amdgpu_device *adev,
 306                                          int instance)
 307 {
 308         uint32_t lo_base_addr, hi_base_addr;
 309         uint64_t config;
 310
 311         config = 0ULL | (0x7ULL) | ((0x46ULL + instance) << 8) | (0x2 << 16);
 312
 313         df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
 314                                       &hi_base_addr);
 315
 316         if ((lo_base_addr == 0) || (hi_base_addr == 0))
 317                 return;
 318
 319         WREG32_PCIE(lo_base_addr, 0UL);
 320         WREG32_PCIE(hi_base_addr, 0UL);
 321 }
 322
 323 /*
 324  * add xgmi link counters
 325  *
 326  * @adev -> amdgpu device
 327  * @instance-> currently cake has 2 links to poll on vega20
 328  *
 329  */
 330
 331 static int df_v3_6_add_xgmi_link_cntr(struct amdgpu_device *adev,
 332                                       int instance)
 333 {
 334         uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
 335         uint64_t config;
 336         int ret, is_assigned;
 337
 338         if (instance < 0 || instance > 1)
 339                 return -EINVAL;
 340
 341         config = GET_INSTANCE_CONFIG(instance);
 342
 343         ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
 344
 345         if (ret || is_assigned)
 346                 return ret;
 347
 348         df_v3_6_pmc_get_ctrl_settings(adev,
 349                         config,
 350                         &lo_base_addr,
 351                         &hi_base_addr,
 352                         &lo_val,
 353                         &hi_val);
 354
 355         WREG32_PCIE(lo_base_addr, lo_val);
 356         WREG32_PCIE(hi_base_addr, hi_val);
 357
 358         return ret;
 359 }
 360
 361
 362 /*
 363  * start xgmi link counters
 364  *
 365  * @adev -> amdgpu device
 366  * @instance-> currently cake has 2 links to poll on vega20
 367  * @is_enable -> either resume or assign event via df perfmon
 368  *
 369  */
 370
 371 static int df_v3_6_start_xgmi_link_cntr(struct amdgpu_device *adev,
 372                                         int instance,
 373                                         int is_enable)
 374 {
 375         uint32_t lo_base_addr, hi_base_addr, lo_val;
 376         uint64_t config;
 377         int ret;
 378
 379         if (instance < 0 || instance > 1)
 380                 return -EINVAL;
 381
 382         if (is_enable) {
 383
 384                 ret = df_v3_6_add_xgmi_link_cntr(adev, instance);
 385
 386                 if (ret)
 387                         return ret;
 388
 389         } else {
 390
 391                 config = GET_INSTANCE_CONFIG(instance);
 392
 393                 df_v3_6_pmc_get_ctrl_settings(adev,
 394                                 config,
 395                                 &lo_base_addr,
 396                                 &hi_base_addr,
 397                                 NULL,
 398                                 NULL);
 399
 400                 if (lo_base_addr == 0)
 401                         return -EINVAL;
 402
 403                 lo_val = RREG32_PCIE(lo_base_addr);
 404
 405                 WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
 406
 407                 ret = 0;
 408         }
 409
 410         return ret;
 411
 412 }
 413
 414 /*
 415  * start xgmi link counters
 416  *
 417  * @adev -> amdgpu device
 418  * @instance-> currently cake has 2 links to poll on vega20
 419  * @is_enable -> either pause or unassign event via df perfmon
 420  *
 421  */
 422
 423 static int df_v3_6_stop_xgmi_link_cntr(struct amdgpu_device *adev,
 424                                        int instance,
 425                                        int is_disable)
 426 {
 427
 428         uint32_t lo_base_addr, hi_base_addr, lo_val;
 429         uint64_t config;
 430
 431         config = GET_INSTANCE_CONFIG(instance);
 432
 433         if (is_disable) {
 434                 df_v3_6_reset_xgmi_link_cntr(adev, instance);
 435                 df_v3_6_pmc_release_cntr(adev, config);
 436         } else {
 437
 438                 df_v3_6_pmc_get_ctrl_settings(adev,
 439                                 config,
 440                                 &lo_base_addr,
 441                                 &hi_base_addr,
 442                                 NULL,
 443                                 NULL);
 444
 445                 if ((lo_base_addr == 0) || (hi_base_addr == 0))
 446                         return -EINVAL;
 447
 448                 lo_val = RREG32_PCIE(lo_base_addr);
 449
 450                 WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
 451         }
 452
 453         return 0;
 454 }
 455
 456 static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
 457                              int is_enable)
 458 {
 459         int xgmi_tx_link, ret = 0;
 460
 461         switch (adev->asic_type) {
 462         case CHIP_VEGA20:
 463                 xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
 464                                         : (IS_DF_XGMI_1_TX(config) ? 1 : -1);
 465
 466                 if (xgmi_tx_link >= 0)
 467                         ret = df_v3_6_start_xgmi_link_cntr(adev, xgmi_tx_link,
 468                                                       is_enable);
 469
 470                 if (ret)
 471                         return ret;
 472
 473                 ret = 0;
 474                 break;
 475         default:
 476                 break;
 477         }
 478
 479         return ret;
 480 }
 481
 482 static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
 483                             int is_disable)
 484 {
 485         int xgmi_tx_link, ret = 0;
 486
 487         switch (adev->asic_type) {
 488         case CHIP_VEGA20:
 489                         xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
 490                                 : (IS_DF_XGMI_1_TX(config) ? 1 : -1);
 491
 492                         if (xgmi_tx_link >= 0) {
 493                                 ret = df_v3_6_stop_xgmi_link_cntr(adev,
 494                                                                   xgmi_tx_link,
 495                                                                   is_disable);
 496                                 if (ret)
 497                                         return ret;
 498                         }
 499
 500                         ret = 0;
 501                         break;
 502         default:
 503                 break;
 504         }
 505
 506         return ret;
 507 }
 508
 509 static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
 510                                   uint64_t config,
 511                                   uint64_t *count)
 512 {
 513
 514         int xgmi_tx_link;
 515
 516         switch (adev->asic_type) {
 517         case CHIP_VEGA20:
 518                 xgmi_tx_link = IS_DF_XGMI_0_TX(config) ? 0
 519                                         : (IS_DF_XGMI_1_TX(config) ? 1 : -1);
 520
 521                 if (xgmi_tx_link >= 0) {
 522                         df_v3_6_reset_xgmi_link_cntr(adev, xgmi_tx_link);
 523                         df_v3_6_get_xgmi_link_cntr(adev, xgmi_tx_link, count);
 524                 }
 525
 526                 break;
 527         default:
 528                 break;
 529         }
 530
 531 }
 532
 533 const struct amdgpu_df_funcs df_v3_6_funcs = {
 534         .init = df_v3_6_init,
 535         .enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
 536         .get_fb_channel_number = df_v3_6_get_fb_channel_number,
 537         .get_hbm_channel_number = df_v3_6_get_hbm_channel_number,
 538         .update_medium_grain_clock_gating =
 539                         df_v3_6_update_medium_grain_clock_gating,
 540         .get_clockgating_state = df_v3_6_get_clockgating_state,
 541         .pmc_start = df_v3_6_pmc_start,
 542         .pmc_stop = df_v3_6_pmc_stop,
 543         .pmc_get_count = df_v3_6_pmc_get_count
 544 };