drivers/gpu/drm/amd/amdgpu/df_v3_6.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "amdgpu.h"
  24 #include "df_v3_6.h"
  25
  26 #include "df/df_3_6_default.h"
  27 #include "df/df_3_6_offset.h"
  28 #include "df/df_3_6_sh_mask.h"
  29
  30 static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
  31                                        16, 32, 0, 0, 0, 2, 4, 8};
  32
  33 /* init df format attrs */
  34 AMDGPU_PMU_ATTR(event,          "config:0-7");
  35 AMDGPU_PMU_ATTR(instance,       "config:8-15");
  36 AMDGPU_PMU_ATTR(umask,          "config:16-23");
  37
  38 /* df format attributes  */
  39 static struct attribute *df_v3_6_format_attrs[] = {
  40         &pmu_attr_event.attr,
  41         &pmu_attr_instance.attr,
  42         &pmu_attr_umask.attr,
  43         NULL
  44 };
  45
  46 /* df format attribute group */
  47 static struct attribute_group df_v3_6_format_attr_group = {
  48         .name = "format",
  49         .attrs = df_v3_6_format_attrs,
  50 };
  51
  52 /* df event attrs */
  53 AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
  54                       "event=0x7,instance=0x46,umask=0x2");
  55 AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
  56                       "event=0x7,instance=0x47,umask=0x2");
  57 AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
  58                       "event=0x7,instance=0x46,umask=0x4");
  59 AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
  60                       "event=0x7,instance=0x47,umask=0x4");
  61 AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
  62                       "event=0xb,instance=0x46,umask=0x4");
  63 AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
  64                       "event=0xb,instance=0x47,umask=0x4");
  65 AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
  66                       "event=0xb,instance=0x46,umask=0x8");
  67 AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
  68                       "event=0xb,instance=0x47,umask=0x8");
  69
  70 /* df event attributes  */
  71 static struct attribute *df_v3_6_event_attrs[] = {
  72         &pmu_attr_cake0_pcsout_txdata.attr,
  73         &pmu_attr_cake1_pcsout_txdata.attr,
  74         &pmu_attr_cake0_pcsout_txmeta.attr,
  75         &pmu_attr_cake1_pcsout_txmeta.attr,
  76         &pmu_attr_cake0_ftiinstat_reqalloc.attr,
  77         &pmu_attr_cake1_ftiinstat_reqalloc.attr,
  78         &pmu_attr_cake0_ftiinstat_rspalloc.attr,
  79         &pmu_attr_cake1_ftiinstat_rspalloc.attr,
  80         NULL
  81 };
  82
  83 /* df event attribute group */
  84 static struct attribute_group df_v3_6_event_attr_group = {
  85         .name = "events",
  86         .attrs = df_v3_6_event_attrs
  87 };
  88
  89 /* df event attr groups  */
  90 const struct attribute_group *df_v3_6_attr_groups[] = {
  91                 &df_v3_6_format_attr_group,
  92                 &df_v3_6_event_attr_group,
  93                 NULL
  94 };
  95
  96 static uint64_t df_v3_6_get_fica(struct amdgpu_device *adev,
  97                                  uint32_t ficaa_val)
  98 {
  99         unsigned long flags, address, data;
 100         uint32_t ficadl_val, ficadh_val;
 101
 102         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 103         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 104
 105         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 106         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessAddress3);
 107         WREG32(data, ficaa_val);
 108
 109         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataLo3);
 110         ficadl_val = RREG32(data);
 111
 112         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3);
 113         ficadh_val = RREG32(data);
 114
 115         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 116
 117         return (((ficadh_val & 0xFFFFFFFFFFFFFFFF) << 32) | ficadl_val);
 118 }
 119
 120 static void df_v3_6_set_fica(struct amdgpu_device *adev, uint32_t ficaa_val,
 121                              uint32_t ficadl_val, uint32_t ficadh_val)
 122 {
 123         unsigned long flags, address, data;
 124
 125         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 126         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 127
 128         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 129         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessAddress3);
 130         WREG32(data, ficaa_val);
 131
 132         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataLo3);
 133         WREG32(data, ficadl_val);
 134
 135         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3);
 136         WREG32(data, ficadh_val);
 137
 138         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 139 }
 140
 141 /*
 142  * df_v3_6_perfmon_rreg - read perfmon lo and hi
 143  *
 144  * required to be atomic.  no mmio method provided so subsequent reads for lo
 145  * and hi require to preserve df finite state machine
 146  */
 147 static void df_v3_6_perfmon_rreg(struct amdgpu_device *adev,
 148                             uint32_t lo_addr, uint32_t *lo_val,
 149                             uint32_t hi_addr, uint32_t *hi_val)
 150 {
 151         unsigned long flags, address, data;
 152
 153         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 154         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 155
 156         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 157         WREG32(address, lo_addr);
 158         *lo_val = RREG32(data);
 159         WREG32(address, hi_addr);
 160         *hi_val = RREG32(data);
 161         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 162 }
 163
 164 /*
 165  * df_v3_6_perfmon_wreg - write to perfmon lo and hi
 166  *
 167  * required to be atomic.  no mmio method provided so subsequent reads after
 168  * data writes cannot occur to preserve data fabrics finite state machine.
 169  */
 170 static void df_v3_6_perfmon_wreg(struct amdgpu_device *adev, uint32_t lo_addr,
 171                             uint32_t lo_val, uint32_t hi_addr, uint32_t hi_val)
 172 {
 173         unsigned long flags, address, data;
 174
 175         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 176         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 177
 178         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 179         WREG32(address, lo_addr);
 180         WREG32(data, lo_val);
 181         WREG32(address, hi_addr);
 182         WREG32(data, hi_val);
 183         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 184 }
 185
 186 /* get the number of df counters available */
 187 static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev,
 188                 struct device_attribute *attr,
 189                 char *buf)
 190 {
 191         struct amdgpu_device *adev;
 192         struct drm_device *ddev;
 193         int i, count;
 194
 195         ddev = dev_get_drvdata(dev);
 196         adev = ddev->dev_private;
 197         count = 0;
 198
 199         for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
 200                 if (adev->df_perfmon_config_assign_mask[i] == 0)
 201                         count++;
 202         }
 203
 204         return snprintf(buf, PAGE_SIZE, "%i\n", count);
 205 }
 206
 207 /* device attr for available perfmon counters */
 208 static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
 209
 210 /* init perfmons */
 211 static void df_v3_6_sw_init(struct amdgpu_device *adev)
 212 {
 213         int i, ret;
 214
 215         ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
 216         if (ret)
 217                 DRM_ERROR("failed to create file for available df counters\n");
 218
 219         for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
 220                 adev->df_perfmon_config_assign_mask[i] = 0;
 221 }
 222
 223 static void df_v3_6_sw_fini(struct amdgpu_device *adev)
 224 {
 225
 226         device_remove_file(adev->dev, &dev_attr_df_cntr_avail);
 227
 228 }
 229
 230 static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
 231                                           bool enable)
 232 {
 233         u32 tmp;
 234
 235         if (enable) {
 236                 tmp = RREG32_SOC15(DF, 0, mmFabricConfigAccessControl);
 237                 tmp &= ~FabricConfigAccessControl__CfgRegInstAccEn_MASK;
 238                 WREG32_SOC15(DF, 0, mmFabricConfigAccessControl, tmp);
 239         } else
 240                 WREG32_SOC15(DF, 0, mmFabricConfigAccessControl,
 241                              mmFabricConfigAccessControl_DEFAULT);
 242 }
 243
 244 static u32 df_v3_6_get_fb_channel_number(struct amdgpu_device *adev)
 245 {
 246         u32 tmp;
 247
 248         tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DramBaseAddress0);
 249         tmp &= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;
 250         tmp >>= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
 251
 252         return tmp;
 253 }
 254
 255 static u32 df_v3_6_get_hbm_channel_number(struct amdgpu_device *adev)
 256 {
 257         int fb_channel_number;
 258
 259         fb_channel_number = adev->df_funcs->get_fb_channel_number(adev);
 260         if (fb_channel_number >= ARRAY_SIZE(df_v3_6_channel_number))
 261                 fb_channel_number = 0;
 262
 263         return df_v3_6_channel_number[fb_channel_number];
 264 }
 265
 266 static void df_v3_6_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 267                                                      bool enable)
 268 {
 269         u32 tmp;
 270
 271         /* Put DF on broadcast mode */
 272         adev->df_funcs->enable_broadcast_mode(adev, true);
 273
 274         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DF_MGCG)) {
 275                 tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
 276                 tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
 277                 tmp |= DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY;
 278                 WREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater, tmp);
 279         } else {
 280                 tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
 281                 tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
 282                 tmp |= DF_V3_6_MGCG_DISABLE;
 283                 WREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater, tmp);
 284         }
 285
 286         /* Exit broadcast mode */
 287         adev->df_funcs->enable_broadcast_mode(adev, false);
 288 }
 289
 290 static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
 291                                           u32 *flags)
 292 {
 293         u32 tmp;
 294
 295         /* AMD_CG_SUPPORT_DF_MGCG */
 296         tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
 297         if (tmp & DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY)
 298                 *flags |= AMD_CG_SUPPORT_DF_MGCG;
 299 }
 300
 301 /* get assigned df perfmon ctr as int */
 302 static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
 303                                       uint64_t config)
 304 {
 305         int i;
 306
 307         for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
 308                 if ((config & 0x0FFFFFFUL) ==
 309                                         adev->df_perfmon_config_assign_mask[i])
 310                         return i;
 311         }
 312
 313         return -EINVAL;
 314 }
 315
 316 /* get address based on counter assignment */
 317 static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
 318                                  uint64_t config,
 319                                  int is_ctrl,
 320                                  uint32_t *lo_base_addr,
 321                                  uint32_t *hi_base_addr)
 322 {
 323         int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 324
 325         if (target_cntr < 0)
 326                 return;
 327
 328         switch (target_cntr) {
 329
 330         case 0:
 331                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo0 : smnPerfMonCtrLo0;
 332                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi0 : smnPerfMonCtrHi0;
 333                 break;
 334         case 1:
 335                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo1 : smnPerfMonCtrLo1;
 336                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi1 : smnPerfMonCtrHi1;
 337                 break;
 338         case 2:
 339                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo2 : smnPerfMonCtrLo2;
 340                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi2 : smnPerfMonCtrHi2;
 341                 break;
 342         case 3:
 343                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo3 : smnPerfMonCtrLo3;
 344                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi3 : smnPerfMonCtrHi3;
 345                 break;
 346
 347         }
 348
 349 }
 350
 351 /* get read counter address */
 352 static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
 353                                           uint64_t config,
 354                                           uint32_t *lo_base_addr,
 355                                           uint32_t *hi_base_addr)
 356 {
 357         df_v3_6_pmc_get_addr(adev, config, 0, lo_base_addr, hi_base_addr);
 358 }
 359
 360 /* get control counter settings i.e. address and values to set */
 361 static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
 362                                           uint64_t config,
 363                                           uint32_t *lo_base_addr,
 364                                           uint32_t *hi_base_addr,
 365                                           uint32_t *lo_val,
 366                                           uint32_t *hi_val)
 367 {
 368
 369         uint32_t eventsel, instance, unitmask;
 370         uint32_t instance_10, instance_5432, instance_76;
 371
 372         df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
 373
 374         if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
 375                 DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
 376                                 *lo_base_addr, *hi_base_addr);
 377                 return -ENXIO;
 378         }
 379
 380         eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
 381         unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
 382         instance = DF_V3_6_GET_INSTANCE(config);
 383
 384         instance_10 = instance & 0x3;
 385         instance_5432 = (instance >> 2) & 0xf;
 386         instance_76 = (instance >> 6) & 0x3;
 387
 388         *lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel | (1 << 22);
 389         *hi_val = (instance_76 << 29) | instance_5432;
 390
 391         DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
 392                 config, *lo_base_addr, *hi_base_addr, *lo_val, *hi_val);
 393
 394         return 0;
 395 }
 396
 397 /* add df performance counters for read */
 398 static int df_v3_6_pmc_add_cntr(struct amdgpu_device *adev,
 399                                    uint64_t config)
 400 {
 401         int i, target_cntr;
 402
 403         target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 404
 405         if (target_cntr >= 0)
 406                 return 0;
 407
 408         for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
 409                 if (adev->df_perfmon_config_assign_mask[i] == 0U) {
 410                         adev->df_perfmon_config_assign_mask[i] =
 411                                                         config & 0x0FFFFFFUL;
 412                         return 0;
 413                 }
 414         }
 415
 416         return -ENOSPC;
 417 }
 418
 419 /* release performance counter */
 420 static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
 421                                      uint64_t config)
 422 {
 423         int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 424
 425         if (target_cntr >= 0)
 426                 adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
 427 }
 428
 429
 430 static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
 431                                          uint64_t config)
 432 {
 433         uint32_t lo_base_addr, hi_base_addr;
 434
 435         df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
 436                                       &hi_base_addr);
 437
 438         if ((lo_base_addr == 0) || (hi_base_addr == 0))
 439                 return;
 440
 441         df_v3_6_perfmon_wreg(adev, lo_base_addr, 0, hi_base_addr, 0);
 442 }
 443
 444 static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
 445                              int is_enable)
 446 {
 447         uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
 448         int ret = 0;
 449
 450         switch (adev->asic_type) {
 451         case CHIP_VEGA20:
 452
 453                 df_v3_6_reset_perfmon_cntr(adev, config);
 454
 455                 if (is_enable) {
 456                         ret = df_v3_6_pmc_add_cntr(adev, config);
 457                 } else {
 458                         ret = df_v3_6_pmc_get_ctrl_settings(adev,
 459                                         config,
 460                                         &lo_base_addr,
 461                                         &hi_base_addr,
 462                                         &lo_val,
 463                                         &hi_val);
 464
 465                         if (ret)
 466                                 return ret;
 467
 468                         df_v3_6_perfmon_wreg(adev, lo_base_addr, lo_val,
 469                                         hi_base_addr, hi_val);
 470                 }
 471
 472                 break;
 473         default:
 474                 break;
 475         }
 476
 477         return ret;
 478 }
 479
 480 static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
 481                             int is_disable)
 482 {
 483         uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
 484         int ret = 0;
 485
 486         switch (adev->asic_type) {
 487         case CHIP_VEGA20:
 488                 ret = df_v3_6_pmc_get_ctrl_settings(adev,
 489                         config,
 490                         &lo_base_addr,
 491                         &hi_base_addr,
 492                         &lo_val,
 493                         &hi_val);
 494
 495                 if (ret)
 496                         return ret;
 497
 498                 df_v3_6_perfmon_wreg(adev, lo_base_addr, 0, hi_base_addr, 0);
 499
 500                 if (is_disable)
 501                         df_v3_6_pmc_release_cntr(adev, config);
 502
 503                 break;
 504         default:
 505                 break;
 506         }
 507
 508         return ret;
 509 }
 510
 511 static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
 512                                   uint64_t config,
 513                                   uint64_t *count)
 514 {
 515         uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
 516         *count = 0;
 517
 518         switch (adev->asic_type) {
 519         case CHIP_VEGA20:
 520
 521                 df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
 522                                       &hi_base_addr);
 523
 524                 if ((lo_base_addr == 0) || (hi_base_addr == 0))
 525                         return;
 526
 527                 df_v3_6_perfmon_rreg(adev, lo_base_addr, &lo_val,
 528                                 hi_base_addr, &hi_val);
 529
 530                 *count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
 531
 532                 if (*count >= DF_V3_6_PERFMON_OVERFLOW)
 533                         *count = 0;
 534
 535                 DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
 536                          config, lo_base_addr, hi_base_addr, lo_val, hi_val);
 537
 538                 break;
 539
 540         default:
 541                 break;
 542         }
 543 }
 544
 545 const struct amdgpu_df_funcs df_v3_6_funcs = {
 546         .sw_init = df_v3_6_sw_init,
 547         .sw_fini = df_v3_6_sw_fini,
 548         .enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
 549         .get_fb_channel_number = df_v3_6_get_fb_channel_number,
 550         .get_hbm_channel_number = df_v3_6_get_hbm_channel_number,
 551         .update_medium_grain_clock_gating =
 552                         df_v3_6_update_medium_grain_clock_gating,
 553         .get_clockgating_state = df_v3_6_get_clockgating_state,
 554         .pmc_start = df_v3_6_pmc_start,
 555         .pmc_stop = df_v3_6_pmc_stop,
 556         .pmc_get_count = df_v3_6_pmc_get_count,
 557         .get_fica = df_v3_6_get_fica,
 558         .set_fica = df_v3_6_set_fica
 559 };