drivers/gpu/drm/i915/i915_pmu.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2017-2018 Intel Corporation
   5  */
   6
   7 #include <linux/pm_runtime.h>
   8
   9 #include "gt/intel_engine.h"
  10 #include "gt/intel_engine_pm.h"
  11 #include "gt/intel_engine_regs.h"
  12 #include "gt/intel_engine_user.h"
  13 #include "gt/intel_gt.h"
  14 #include "gt/intel_gt_pm.h"
  15 #include "gt/intel_gt_regs.h"
  16 #include "gt/intel_rc6.h"
  17 #include "gt/intel_rps.h"
  18
  19 #include "i915_drv.h"
  20 #include "i915_pmu.h"
  21
  22 /* Frequency for the sampling timer for events which need it. */
  23 #define FREQUENCY 200
  24 #define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)
  25
  26 #define ENGINE_SAMPLE_MASK \
  27         (BIT(I915_SAMPLE_BUSY) | \
  28          BIT(I915_SAMPLE_WAIT) | \
  29          BIT(I915_SAMPLE_SEMA))
  30
  31 static cpumask_t i915_pmu_cpumask;
  32 static unsigned int i915_pmu_target_cpu = -1;
  33
  34 static struct i915_pmu *event_to_pmu(struct perf_event *event)
  35 {
  36         return container_of(event->pmu, struct i915_pmu, base);
  37 }
  38
  39 static struct drm_i915_private *pmu_to_i915(struct i915_pmu *pmu)
  40 {
  41         return container_of(pmu, struct drm_i915_private, pmu);
  42 }
  43
  44 static u8 engine_config_sample(u64 config)
  45 {
  46         return config & I915_PMU_SAMPLE_MASK;
  47 }
  48
  49 static u8 engine_event_sample(struct perf_event *event)
  50 {
  51         return engine_config_sample(event->attr.config);
  52 }
  53
  54 static u8 engine_event_class(struct perf_event *event)
  55 {
  56         return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
  57 }
  58
  59 static u8 engine_event_instance(struct perf_event *event)
  60 {
  61         return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
  62 }
  63
  64 static bool is_engine_config(const u64 config)
  65 {
  66         return config < __I915_PMU_OTHER(0);
  67 }
  68
  69 static unsigned int config_gt_id(const u64 config)
  70 {
  71         return config >> __I915_PMU_GT_SHIFT;
  72 }
  73
  74 static u64 config_counter(const u64 config)
  75 {
  76         return config & ~(~0ULL << __I915_PMU_GT_SHIFT);
  77 }
  78
  79 static unsigned int other_bit(const u64 config)
  80 {
  81         unsigned int val;
  82
  83         switch (config_counter(config)) {
  84         case I915_PMU_ACTUAL_FREQUENCY:
  85                 val =  __I915_PMU_ACTUAL_FREQUENCY_ENABLED;
  86                 break;
  87         case I915_PMU_REQUESTED_FREQUENCY:
  88                 val = __I915_PMU_REQUESTED_FREQUENCY_ENABLED;
  89                 break;
  90         case I915_PMU_RC6_RESIDENCY:
  91                 val = __I915_PMU_RC6_RESIDENCY_ENABLED;
  92                 break;
  93         default:
  94                 /*
  95                  * Events that do not require sampling, or tracking state
  96                  * transitions between enabled and disabled can be ignored.
  97                  */
  98                 return -1;
  99         }
 100
 101         return I915_ENGINE_SAMPLE_COUNT +
 102                config_gt_id(config) * __I915_PMU_TRACKED_EVENT_COUNT +
 103                val;
 104 }
 105
 106 static unsigned int config_bit(const u64 config)
 107 {
 108         if (is_engine_config(config))
 109                 return engine_config_sample(config);
 110         else
 111                 return other_bit(config);
 112 }
 113
 114 static u32 config_mask(const u64 config)
 115 {
 116         unsigned int bit = config_bit(config);
 117
 118         if (__builtin_constant_p(config))
 119                 BUILD_BUG_ON(bit >
 120                              BITS_PER_TYPE(typeof_member(struct i915_pmu,
 121                                                          enable)) - 1);
 122         else
 123                 WARN_ON_ONCE(bit >
 124                              BITS_PER_TYPE(typeof_member(struct i915_pmu,
 125                                                          enable)) - 1);
 126
 127         return BIT(config_bit(config));
 128 }
 129
 130 static bool is_engine_event(struct perf_event *event)
 131 {
 132         return is_engine_config(event->attr.config);
 133 }
 134
 135 static unsigned int event_bit(struct perf_event *event)
 136 {
 137         return config_bit(event->attr.config);
 138 }
 139
 140 static u32 frequency_enabled_mask(void)
 141 {
 142         unsigned int i;
 143         u32 mask = 0;
 144
 145         for (i = 0; i < I915_PMU_MAX_GT; i++)
 146                 mask |= config_mask(__I915_PMU_ACTUAL_FREQUENCY(i)) |
 147                         config_mask(__I915_PMU_REQUESTED_FREQUENCY(i));
 148
 149         return mask;
 150 }
 151
 152 static bool pmu_needs_timer(struct i915_pmu *pmu)
 153 {
 154         struct drm_i915_private *i915 = pmu_to_i915(pmu);
 155         u32 enable;
 156
 157         /*
 158          * Only some counters need the sampling timer.
 159          *
 160          * We start with a bitmask of all currently enabled events.
 161          */
 162         enable = pmu->enable;
 163
 164         /*
 165          * Mask out all the ones which do not need the timer, or in
 166          * other words keep all the ones that could need the timer.
 167          */
 168         enable &= frequency_enabled_mask() | ENGINE_SAMPLE_MASK;
 169
 170         /*
 171          * Also there is software busyness tracking available we do not
 172          * need the timer for I915_SAMPLE_BUSY counter.
 173          */
 174         if (i915->caps.scheduler & I915_SCHEDULER_CAP_ENGINE_BUSY_STATS)
 175                 enable &= ~BIT(I915_SAMPLE_BUSY);
 176
 177         /*
 178          * If some bits remain it means we need the sampling timer running.
 179          */
 180         return enable;
 181 }
 182
 183 static u64 __get_rc6(struct intel_gt *gt)
 184 {
 185         struct drm_i915_private *i915 = gt->i915;
 186         u64 val;
 187
 188         val = intel_rc6_residency_ns(&gt->rc6, INTEL_RC6_RES_RC6);
 189
 190         if (HAS_RC6p(i915))
 191                 val += intel_rc6_residency_ns(&gt->rc6, INTEL_RC6_RES_RC6p);
 192
 193         if (HAS_RC6pp(i915))
 194                 val += intel_rc6_residency_ns(&gt->rc6, INTEL_RC6_RES_RC6pp);
 195
 196         return val;
 197 }
 198
 199 static inline s64 ktime_since_raw(const ktime_t kt)
 200 {
 201         return ktime_to_ns(ktime_sub(ktime_get_raw(), kt));
 202 }
 203
 204 static u64 read_sample(struct i915_pmu *pmu, unsigned int gt_id, int sample)
 205 {
 206         return pmu->sample[gt_id][sample].cur;
 207 }
 208
 209 static void
 210 store_sample(struct i915_pmu *pmu, unsigned int gt_id, int sample, u64 val)
 211 {
 212         pmu->sample[gt_id][sample].cur = val;
 213 }
 214
 215 static void
 216 add_sample_mult(struct i915_pmu *pmu, unsigned int gt_id, int sample, u32 val, u32 mul)
 217 {
 218         pmu->sample[gt_id][sample].cur += mul_u32_u32(val, mul);
 219 }
 220
 221 static u64 get_rc6(struct intel_gt *gt)
 222 {
 223         struct drm_i915_private *i915 = gt->i915;
 224         const unsigned int gt_id = gt->info.id;
 225         struct i915_pmu *pmu = &i915->pmu;
 226         intel_wakeref_t wakeref;
 227         unsigned long flags;
 228         u64 val;
 229
 230         wakeref = intel_gt_pm_get_if_awake(gt);
 231         if (wakeref) {
 232                 val = __get_rc6(gt);
 233                 intel_gt_pm_put_async(gt, wakeref);
 234         }
 235
 236         spin_lock_irqsave(&pmu->lock, flags);
 237
 238         if (wakeref) {
 239                 store_sample(pmu, gt_id, __I915_SAMPLE_RC6, val);
 240         } else {
 241                 /*
 242                  * We think we are runtime suspended.
 243                  *
 244                  * Report the delta from when the device was suspended to now,
 245                  * on top of the last known real value, as the approximated RC6
 246                  * counter value.
 247                  */
 248                 val = ktime_since_raw(pmu->sleep_last[gt_id]);
 249                 val += read_sample(pmu, gt_id, __I915_SAMPLE_RC6);
 250         }
 251
 252         if (val < read_sample(pmu, gt_id, __I915_SAMPLE_RC6_LAST_REPORTED))
 253                 val = read_sample(pmu, gt_id, __I915_SAMPLE_RC6_LAST_REPORTED);
 254         else
 255                 store_sample(pmu, gt_id, __I915_SAMPLE_RC6_LAST_REPORTED, val);
 256
 257         spin_unlock_irqrestore(&pmu->lock, flags);
 258
 259         return val;
 260 }
 261
 262 static void init_rc6(struct i915_pmu *pmu)
 263 {
 264         struct drm_i915_private *i915 = pmu_to_i915(pmu);
 265         struct intel_gt *gt;
 266         unsigned int i;
 267
 268         for_each_gt(gt, i915, i) {
 269                 intel_wakeref_t wakeref;
 270
 271                 with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
 272                         u64 val = __get_rc6(gt);
 273
 274                         store_sample(pmu, i, __I915_SAMPLE_RC6, val);
 275                         store_sample(pmu, i, __I915_SAMPLE_RC6_LAST_REPORTED,
 276                                      val);
 277                         pmu->sleep_last[i] = ktime_get_raw();
 278                 }
 279         }
 280 }
 281
 282 static void park_rc6(struct intel_gt *gt)
 283 {
 284         struct i915_pmu *pmu = &gt->i915->pmu;
 285
 286         store_sample(pmu, gt->info.id, __I915_SAMPLE_RC6, __get_rc6(gt));
 287         pmu->sleep_last[gt->info.id] = ktime_get_raw();
 288 }
 289
 290 static void __i915_pmu_maybe_start_timer(struct i915_pmu *pmu)
 291 {
 292         if (!pmu->timer_enabled && pmu_needs_timer(pmu)) {
 293                 pmu->timer_enabled = true;
 294                 pmu->timer_last = ktime_get();
 295                 hrtimer_start_range_ns(&pmu->timer,
 296                                        ns_to_ktime(PERIOD), 0,
 297                                        HRTIMER_MODE_REL_PINNED);
 298         }
 299 }
 300
 301 void i915_pmu_gt_parked(struct intel_gt *gt)
 302 {
 303         struct i915_pmu *pmu = &gt->i915->pmu;
 304
 305         if (!pmu->registered)
 306                 return;
 307
 308         spin_lock_irq(&pmu->lock);
 309
 310         park_rc6(gt);
 311
 312         /*
 313          * Signal sampling timer to stop if only engine events are enabled and
 314          * GPU went idle.
 315          */
 316         pmu->unparked &= ~BIT(gt->info.id);
 317         if (pmu->unparked == 0)
 318                 pmu->timer_enabled = false;
 319
 320         spin_unlock_irq(&pmu->lock);
 321 }
 322
 323 void i915_pmu_gt_unparked(struct intel_gt *gt)
 324 {
 325         struct i915_pmu *pmu = &gt->i915->pmu;
 326
 327         if (!pmu->registered)
 328                 return;
 329
 330         spin_lock_irq(&pmu->lock);
 331
 332         /*
 333          * Re-enable sampling timer when GPU goes active.
 334          */
 335         if (pmu->unparked == 0)
 336                 __i915_pmu_maybe_start_timer(pmu);
 337
 338         pmu->unparked |= BIT(gt->info.id);
 339
 340         spin_unlock_irq(&pmu->lock);
 341 }
 342
 343 static void
 344 add_sample(struct i915_pmu_sample *sample, u32 val)
 345 {
 346         sample->cur += val;
 347 }
 348
 349 static bool exclusive_mmio_access(const struct drm_i915_private *i915)
 350 {
 351         /*
 352          * We have to avoid concurrent mmio cache line access on gen7 or
 353          * risk a machine hang. For a fun history lesson dig out the old
 354          * userspace intel_gpu_top and run it on Ivybridge or Haswell!
 355          */
 356         return GRAPHICS_VER(i915) == 7;
 357 }
 358
 359 static void gen3_engine_sample(struct intel_engine_cs *engine, unsigned int period_ns)
 360 {
 361         struct intel_engine_pmu *pmu = &engine->pmu;
 362         bool busy;
 363         u32 val;
 364
 365         val = ENGINE_READ_FW(engine, RING_CTL);
 366         if (val == 0) /* powerwell off => engine idle */
 367                 return;
 368
 369         if (val & RING_WAIT)
 370                 add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns);
 371         if (val & RING_WAIT_SEMAPHORE)
 372                 add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns);
 373
 374         /* No need to sample when busy stats are supported. */
 375         if (intel_engine_supports_stats(engine))
 376                 return;
 377
 378         /*
 379          * While waiting on a semaphore or event, MI_MODE reports the
 380          * ring as idle. However, previously using the seqno, and with
 381          * execlists sampling, we account for the ring waiting as the
 382          * engine being busy. Therefore, we record the sample as being
 383          * busy if either waiting or !idle.
 384          */
 385         busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT);
 386         if (!busy) {
 387                 val = ENGINE_READ_FW(engine, RING_MI_MODE);
 388                 busy = !(val & MODE_IDLE);
 389         }
 390         if (busy)
 391                 add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns);
 392 }
 393
 394 static void gen2_engine_sample(struct intel_engine_cs *engine, unsigned int period_ns)
 395 {
 396         struct intel_engine_pmu *pmu = &engine->pmu;
 397         u32 tail, head, acthd;
 398
 399         tail = ENGINE_READ_FW(engine, RING_TAIL);
 400         head = ENGINE_READ_FW(engine, RING_HEAD);
 401         acthd = ENGINE_READ_FW(engine, ACTHD);
 402
 403         if (head & HEAD_WAIT_I8XX)
 404                 add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns);
 405
 406         if (head & HEAD_WAIT_I8XX || head != acthd ||
 407             (head & HEAD_ADDR) != (tail & TAIL_ADDR))
 408                 add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns);
 409 }
 410
 411 static void engine_sample(struct intel_engine_cs *engine, unsigned int period_ns)
 412 {
 413         if (GRAPHICS_VER(engine->i915) >= 3)
 414                 gen3_engine_sample(engine, period_ns);
 415         else
 416                 gen2_engine_sample(engine, period_ns);
 417 }
 418
 419 static void
 420 engines_sample(struct intel_gt *gt, unsigned int period_ns)
 421 {
 422         struct drm_i915_private *i915 = gt->i915;
 423         struct intel_engine_cs *engine;
 424         enum intel_engine_id id;
 425         unsigned long flags;
 426
 427         if ((i915->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
 428                 return;
 429
 430         if (!intel_gt_pm_is_awake(gt))
 431                 return;
 432
 433         for_each_engine(engine, gt, id) {
 434                 if (!engine->pmu.enable)
 435                         continue;
 436
 437                 if (!intel_engine_pm_get_if_awake(engine))
 438                         continue;
 439
 440                 if (exclusive_mmio_access(i915)) {
 441                         spin_lock_irqsave(&engine->uncore->lock, flags);
 442                         engine_sample(engine, period_ns);
 443                         spin_unlock_irqrestore(&engine->uncore->lock, flags);
 444                 } else {
 445                         engine_sample(engine, period_ns);
 446                 }
 447
 448                 intel_engine_pm_put_async(engine);
 449         }
 450 }
 451
 452 static bool
 453 frequency_sampling_enabled(struct i915_pmu *pmu, unsigned int gt)
 454 {
 455         return pmu->enable &
 456                (config_mask(__I915_PMU_ACTUAL_FREQUENCY(gt)) |
 457                 config_mask(__I915_PMU_REQUESTED_FREQUENCY(gt)));
 458 }
 459
 460 static void
 461 frequency_sample(struct intel_gt *gt, unsigned int period_ns)
 462 {
 463         struct drm_i915_private *i915 = gt->i915;
 464         const unsigned int gt_id = gt->info.id;
 465         struct i915_pmu *pmu = &i915->pmu;
 466         struct intel_rps *rps = &gt->rps;
 467         intel_wakeref_t wakeref;
 468
 469         if (!frequency_sampling_enabled(pmu, gt_id))
 470                 return;
 471
 472         /* Report 0/0 (actual/requested) frequency while parked. */
 473         wakeref = intel_gt_pm_get_if_awake(gt);
 474         if (!wakeref)
 475                 return;
 476
 477         if (pmu->enable & config_mask(__I915_PMU_ACTUAL_FREQUENCY(gt_id))) {
 478                 u32 val;
 479
 480                 /*
 481                  * We take a quick peek here without using forcewake
 482                  * so that we don't perturb the system under observation
 483                  * (forcewake => !rc6 => increased power use). We expect
 484                  * that if the read fails because it is outside of the
 485                  * mmio power well, then it will return 0 -- in which
 486                  * case we assume the system is running at the intended
 487                  * frequency. Fortunately, the read should rarely fail!
 488                  */
 489                 val = intel_rps_read_actual_frequency_fw(rps);
 490                 if (!val)
 491                         val = intel_gpu_freq(rps, rps->cur_freq);
 492
 493                 add_sample_mult(pmu, gt_id, __I915_SAMPLE_FREQ_ACT,
 494                                 val, period_ns / 1000);
 495         }
 496
 497         if (pmu->enable & config_mask(__I915_PMU_REQUESTED_FREQUENCY(gt_id))) {
 498                 add_sample_mult(pmu, gt_id, __I915_SAMPLE_FREQ_REQ,
 499                                 intel_rps_get_requested_frequency(rps),
 500                                 period_ns / 1000);
 501         }
 502
 503         intel_gt_pm_put_async(gt, wakeref);
 504 }
 505
 506 static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer)
 507 {
 508         struct i915_pmu *pmu = container_of(hrtimer, struct i915_pmu, timer);
 509         struct drm_i915_private *i915 = pmu_to_i915(pmu);
 510         unsigned int period_ns;
 511         struct intel_gt *gt;
 512         unsigned int i;
 513         ktime_t now;
 514
 515         if (!READ_ONCE(pmu->timer_enabled))
 516                 return HRTIMER_NORESTART;
 517
 518         now = ktime_get();
 519         period_ns = ktime_to_ns(ktime_sub(now, pmu->timer_last));
 520         pmu->timer_last = now;
 521
 522         /*
 523          * Strictly speaking the passed in period may not be 100% accurate for
 524          * all internal calculation, since some amount of time can be spent on
 525          * grabbing the forcewake. However the potential error from timer call-
 526          * back delay greatly dominates this so we keep it simple.
 527          */
 528
 529         for_each_gt(gt, i915, i) {
 530                 if (!(pmu->unparked & BIT(i)))
 531                         continue;
 532
 533                 engines_sample(gt, period_ns);
 534                 frequency_sample(gt, period_ns);
 535         }
 536
 537         hrtimer_forward(hrtimer, now, ns_to_ktime(PERIOD));
 538
 539         return HRTIMER_RESTART;
 540 }
 541
 542 static void i915_pmu_event_destroy(struct perf_event *event)
 543 {
 544         struct i915_pmu *pmu = event_to_pmu(event);
 545         struct drm_i915_private *i915 = pmu_to_i915(pmu);
 546
 547         drm_WARN_ON(&i915->drm, event->parent);
 548
 549         drm_dev_put(&i915->drm);
 550 }
 551
 552 static int
 553 engine_event_status(struct intel_engine_cs *engine,
 554                     enum drm_i915_pmu_engine_sample sample)
 555 {
 556         switch (sample) {
 557         case I915_SAMPLE_BUSY:
 558         case I915_SAMPLE_WAIT:
 559                 break;
 560         case I915_SAMPLE_SEMA:
 561                 if (GRAPHICS_VER(engine->i915) < 6)
 562                         return -ENODEV;
 563                 break;
 564         default:
 565                 return -ENOENT;
 566         }
 567
 568         return 0;
 569 }
 570
 571 static int
 572 config_status(struct drm_i915_private *i915, u64 config)
 573 {
 574         struct intel_gt *gt = to_gt(i915);
 575
 576         unsigned int gt_id = config_gt_id(config);
 577         unsigned int max_gt_id = HAS_EXTRA_GT_LIST(i915) ? 1 : 0;
 578
 579         if (gt_id > max_gt_id)
 580                 return -ENOENT;
 581
 582         switch (config_counter(config)) {
 583         case I915_PMU_ACTUAL_FREQUENCY:
 584                 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 585                         /* Requires a mutex for sampling! */
 586                         return -ENODEV;
 587                 fallthrough;
 588         case I915_PMU_REQUESTED_FREQUENCY:
 589                 if (GRAPHICS_VER(i915) < 6)
 590                         return -ENODEV;
 591                 break;
 592         case I915_PMU_INTERRUPTS:
 593                 if (gt_id)
 594                         return -ENOENT;
 595                 break;
 596         case I915_PMU_RC6_RESIDENCY:
 597                 if (!gt->rc6.supported)
 598                         return -ENODEV;
 599                 break;
 600         case I915_PMU_SOFTWARE_GT_AWAKE_TIME:
 601                 break;
 602         default:
 603                 return -ENOENT;
 604         }
 605
 606         return 0;
 607 }
 608
 609 static int engine_event_init(struct perf_event *event)
 610 {
 611         struct i915_pmu *pmu = event_to_pmu(event);
 612         struct drm_i915_private *i915 = pmu_to_i915(pmu);
 613         struct intel_engine_cs *engine;
 614
 615         engine = intel_engine_lookup_user(i915, engine_event_class(event),
 616                                           engine_event_instance(event));
 617         if (!engine)
 618                 return -ENODEV;
 619
 620         return engine_event_status(engine, engine_event_sample(event));
 621 }
 622
 623 static int i915_pmu_event_init(struct perf_event *event)
 624 {
 625         struct i915_pmu *pmu = event_to_pmu(event);
 626         struct drm_i915_private *i915 = pmu_to_i915(pmu);
 627         int ret;
 628
 629         if (!pmu->registered)
 630                 return -ENODEV;
 631
 632         if (event->attr.type != event->pmu->type)
 633                 return -ENOENT;
 634
 635         /* unsupported modes and filters */
 636         if (event->attr.sample_period) /* no sampling */
 637                 return -EINVAL;
 638
 639         if (has_branch_stack(event))
 640                 return -EOPNOTSUPP;
 641
 642         if (event->cpu < 0)
 643                 return -EINVAL;
 644
 645         /* only allow running on one cpu at a time */
 646         if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask))
 647                 return -EINVAL;
 648
 649         if (is_engine_event(event))
 650                 ret = engine_event_init(event);
 651         else
 652                 ret = config_status(i915, event->attr.config);
 653         if (ret)
 654                 return ret;
 655
 656         if (!event->parent) {
 657                 drm_dev_get(&i915->drm);
 658                 event->destroy = i915_pmu_event_destroy;
 659         }
 660
 661         return 0;
 662 }
 663
 664 static u64 __i915_pmu_event_read(struct perf_event *event)
 665 {
 666         struct i915_pmu *pmu = event_to_pmu(event);
 667         struct drm_i915_private *i915 = pmu_to_i915(pmu);
 668         u64 val = 0;
 669
 670         if (is_engine_event(event)) {
 671                 u8 sample = engine_event_sample(event);
 672                 struct intel_engine_cs *engine;
 673
 674                 engine = intel_engine_lookup_user(i915,
 675                                                   engine_event_class(event),
 676                                                   engine_event_instance(event));
 677
 678                 if (drm_WARN_ON_ONCE(&i915->drm, !engine)) {
 679                         /* Do nothing */
 680                 } else if (sample == I915_SAMPLE_BUSY &&
 681                            intel_engine_supports_stats(engine)) {
 682                         ktime_t unused;
 683
 684                         val = ktime_to_ns(intel_engine_get_busy_time(engine,
 685                                                                      &unused));
 686                 } else {
 687                         val = engine->pmu.sample[sample].cur;
 688                 }
 689         } else {
 690                 const unsigned int gt_id = config_gt_id(event->attr.config);
 691                 const u64 config = config_counter(event->attr.config);
 692
 693                 switch (config) {
 694                 case I915_PMU_ACTUAL_FREQUENCY:
 695                         val =
 696                            div_u64(read_sample(pmu, gt_id,
 697                                                __I915_SAMPLE_FREQ_ACT),
 698                                    USEC_PER_SEC /* to MHz */);
 699                         break;
 700                 case I915_PMU_REQUESTED_FREQUENCY:
 701                         val =
 702                            div_u64(read_sample(pmu, gt_id,
 703                                                __I915_SAMPLE_FREQ_REQ),
 704                                    USEC_PER_SEC /* to MHz */);
 705                         break;
 706                 case I915_PMU_INTERRUPTS:
 707                         val = READ_ONCE(pmu->irq_count);
 708                         break;
 709                 case I915_PMU_RC6_RESIDENCY:
 710                         val = get_rc6(i915->gt[gt_id]);
 711                         break;
 712                 case I915_PMU_SOFTWARE_GT_AWAKE_TIME:
 713                         val = ktime_to_ns(intel_gt_get_awake_time(to_gt(i915)));
 714                         break;
 715                 }
 716         }
 717
 718         return val;
 719 }
 720
 721 static void i915_pmu_event_read(struct perf_event *event)
 722 {
 723         struct i915_pmu *pmu = event_to_pmu(event);
 724         struct hw_perf_event *hwc = &event->hw;
 725         u64 prev, new;
 726
 727         if (!pmu->registered) {
 728                 event->hw.state = PERF_HES_STOPPED;
 729                 return;
 730         }
 731
 732         prev = local64_read(&hwc->prev_count);
 733         do {
 734                 new = __i915_pmu_event_read(event);
 735         } while (!local64_try_cmpxchg(&hwc->prev_count, &prev, new));
 736
 737         local64_add(new - prev, &event->count);
 738 }
 739
 740 static void i915_pmu_enable(struct perf_event *event)
 741 {
 742         struct i915_pmu *pmu = event_to_pmu(event);
 743         struct drm_i915_private *i915 = pmu_to_i915(pmu);
 744         const unsigned int bit = event_bit(event);
 745         unsigned long flags;
 746
 747         if (bit == -1)
 748                 goto update;
 749
 750         spin_lock_irqsave(&pmu->lock, flags);
 751
 752         /*
 753          * Update the bitmask of enabled events and increment
 754          * the event reference counter.
 755          */
 756         BUILD_BUG_ON(ARRAY_SIZE(pmu->enable_count) != I915_PMU_MASK_BITS);
 757         GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count));
 758         GEM_BUG_ON(pmu->enable_count[bit] == ~0);
 759
 760         pmu->enable |= BIT(bit);
 761         pmu->enable_count[bit]++;
 762
 763         /*
 764          * Start the sampling timer if needed and not already enabled.
 765          */
 766         __i915_pmu_maybe_start_timer(pmu);
 767
 768         /*
 769          * For per-engine events the bitmask and reference counting
 770          * is stored per engine.
 771          */
 772         if (is_engine_event(event)) {
 773                 u8 sample = engine_event_sample(event);
 774                 struct intel_engine_cs *engine;
 775
 776                 engine = intel_engine_lookup_user(i915,
 777                                                   engine_event_class(event),
 778                                                   engine_event_instance(event));
 779
 780                 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) !=
 781                              I915_ENGINE_SAMPLE_COUNT);
 782                 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) !=
 783                              I915_ENGINE_SAMPLE_COUNT);
 784                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
 785                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
 786                 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);
 787
 788                 engine->pmu.enable |= BIT(sample);
 789                 engine->pmu.enable_count[sample]++;
 790         }
 791
 792         spin_unlock_irqrestore(&pmu->lock, flags);
 793
 794 update:
 795         /*
 796          * Store the current counter value so we can report the correct delta
 797          * for all listeners. Even when the event was already enabled and has
 798          * an existing non-zero value.
 799          */
 800         local64_set(&event->hw.prev_count, __i915_pmu_event_read(event));
 801 }
 802
 803 static void i915_pmu_disable(struct perf_event *event)
 804 {
 805         struct i915_pmu *pmu = event_to_pmu(event);
 806         struct drm_i915_private *i915 = pmu_to_i915(pmu);
 807         const unsigned int bit = event_bit(event);
 808         unsigned long flags;
 809
 810         if (bit == -1)
 811                 return;
 812
 813         spin_lock_irqsave(&pmu->lock, flags);
 814
 815         if (is_engine_event(event)) {
 816                 u8 sample = engine_event_sample(event);
 817                 struct intel_engine_cs *engine;
 818
 819                 engine = intel_engine_lookup_user(i915,
 820                                                   engine_event_class(event),
 821                                                   engine_event_instance(event));
 822
 823                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
 824                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
 825                 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);
 826
 827                 /*
 828                  * Decrement the reference count and clear the enabled
 829                  * bitmask when the last listener on an event goes away.
 830                  */
 831                 if (--engine->pmu.enable_count[sample] == 0)
 832                         engine->pmu.enable &= ~BIT(sample);
 833         }
 834
 835         GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count));
 836         GEM_BUG_ON(pmu->enable_count[bit] == 0);
 837         /*
 838          * Decrement the reference count and clear the enabled
 839          * bitmask when the last listener on an event goes away.
 840          */
 841         if (--pmu->enable_count[bit] == 0) {
 842                 pmu->enable &= ~BIT(bit);
 843                 pmu->timer_enabled &= pmu_needs_timer(pmu);
 844         }
 845
 846         spin_unlock_irqrestore(&pmu->lock, flags);
 847 }
 848
 849 static void i915_pmu_event_start(struct perf_event *event, int flags)
 850 {
 851         struct i915_pmu *pmu = event_to_pmu(event);
 852
 853         if (!pmu->registered)
 854                 return;
 855
 856         i915_pmu_enable(event);
 857         event->hw.state = 0;
 858 }
 859
 860 static void i915_pmu_event_stop(struct perf_event *event, int flags)
 861 {
 862         struct i915_pmu *pmu = event_to_pmu(event);
 863
 864         if (!pmu->registered)
 865                 goto out;
 866
 867         if (flags & PERF_EF_UPDATE)
 868                 i915_pmu_event_read(event);
 869
 870         i915_pmu_disable(event);
 871
 872 out:
 873         event->hw.state = PERF_HES_STOPPED;
 874 }
 875
 876 static int i915_pmu_event_add(struct perf_event *event, int flags)
 877 {
 878         struct i915_pmu *pmu = event_to_pmu(event);
 879
 880         if (!pmu->registered)
 881                 return -ENODEV;
 882
 883         if (flags & PERF_EF_START)
 884                 i915_pmu_event_start(event, flags);
 885
 886         return 0;
 887 }
 888
 889 static void i915_pmu_event_del(struct perf_event *event, int flags)
 890 {
 891         i915_pmu_event_stop(event, PERF_EF_UPDATE);
 892 }
 893
 894 static int i915_pmu_event_event_idx(struct perf_event *event)
 895 {
 896         return 0;
 897 }
 898
 899 struct i915_str_attribute {
 900         struct device_attribute attr;
 901         const char *str;
 902 };
 903
 904 static ssize_t i915_pmu_format_show(struct device *dev,
 905                                     struct device_attribute *attr, char *buf)
 906 {
 907         struct i915_str_attribute *eattr;
 908
 909         eattr = container_of(attr, struct i915_str_attribute, attr);
 910         return sprintf(buf, "%s\n", eattr->str);
 911 }
 912
 913 #define I915_PMU_FORMAT_ATTR(_name, _config) \
 914         (&((struct i915_str_attribute[]) { \
 915                 { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \
 916                   .str = _config, } \
 917         })[0].attr.attr)
 918
 919 static struct attribute *i915_pmu_format_attrs[] = {
 920         I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
 921         NULL,
 922 };
 923
 924 static const struct attribute_group i915_pmu_format_attr_group = {
 925         .name = "format",
 926         .attrs = i915_pmu_format_attrs,
 927 };
 928
 929 struct i915_ext_attribute {
 930         struct device_attribute attr;
 931         unsigned long val;
 932 };
 933
 934 static ssize_t i915_pmu_event_show(struct device *dev,
 935                                    struct device_attribute *attr, char *buf)
 936 {
 937         struct i915_ext_attribute *eattr;
 938
 939         eattr = container_of(attr, struct i915_ext_attribute, attr);
 940         return sprintf(buf, "config=0x%lx\n", eattr->val);
 941 }
 942
 943 static ssize_t cpumask_show(struct device *dev,
 944                             struct device_attribute *attr, char *buf)
 945 {
 946         return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask);
 947 }
 948
 949 static DEVICE_ATTR_RO(cpumask);
 950
 951 static struct attribute *i915_cpumask_attrs[] = {
 952         &dev_attr_cpumask.attr,
 953         NULL,
 954 };
 955
 956 static const struct attribute_group i915_pmu_cpumask_attr_group = {
 957         .attrs = i915_cpumask_attrs,
 958 };
 959
 960 #define __event(__counter, __name, __unit) \
 961 { \
 962         .counter = (__counter), \
 963         .name = (__name), \
 964         .unit = (__unit), \
 965         .global = false, \
 966 }
 967
 968 #define __global_event(__counter, __name, __unit) \
 969 { \
 970         .counter = (__counter), \
 971         .name = (__name), \
 972         .unit = (__unit), \
 973         .global = true, \
 974 }
 975
 976 #define __engine_event(__sample, __name) \
 977 { \
 978         .sample = (__sample), \
 979         .name = (__name), \
 980 }
 981
 982 static struct i915_ext_attribute *
 983 add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config)
 984 {
 985         sysfs_attr_init(&attr->attr.attr);
 986         attr->attr.attr.name = name;
 987         attr->attr.attr.mode = 0444;
 988         attr->attr.show = i915_pmu_event_show;
 989         attr->val = config;
 990
 991         return ++attr;
 992 }
 993
 994 static struct perf_pmu_events_attr *
 995 add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name,
 996              const char *str)
 997 {
 998         sysfs_attr_init(&attr->attr.attr);
 999         attr->attr.attr.name = name;
1000         attr->attr.attr.mode = 0444;
1001         attr->attr.show = perf_event_sysfs_show;
1002         attr->event_str = str;
1003
1004         return ++attr;
1005 }
1006
1007 static struct attribute **
1008 create_event_attributes(struct i915_pmu *pmu)
1009 {
1010         struct drm_i915_private *i915 = pmu_to_i915(pmu);
1011         static const struct {
1012                 unsigned int counter;
1013                 const char *name;
1014                 const char *unit;
1015                 bool global;
1016         } events[] = {
1017                 __event(0, "actual-frequency", "M"),
1018                 __event(1, "requested-frequency", "M"),
1019                 __global_event(2, "interrupts", NULL),
1020                 __event(3, "rc6-residency", "ns"),
1021                 __event(4, "software-gt-awake-time", "ns"),
1022         };
1023         static const struct {
1024                 enum drm_i915_pmu_engine_sample sample;
1025                 char *name;
1026         } engine_events[] = {
1027                 __engine_event(I915_SAMPLE_BUSY, "busy"),
1028                 __engine_event(I915_SAMPLE_SEMA, "sema"),
1029                 __engine_event(I915_SAMPLE_WAIT, "wait"),
1030         };
1031         unsigned int count = 0;
1032         struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter;
1033         struct i915_ext_attribute *i915_attr = NULL, *i915_iter;
1034         struct attribute **attr = NULL, **attr_iter;
1035         struct intel_engine_cs *engine;
1036         struct intel_gt *gt;
1037         unsigned int i, j;
1038
1039         /* Count how many counters we will be exposing. */
1040         for_each_gt(gt, i915, j) {
1041                 for (i = 0; i < ARRAY_SIZE(events); i++) {
1042                         u64 config = ___I915_PMU_OTHER(j, events[i].counter);
1043
1044                         if (!config_status(i915, config))
1045                                 count++;
1046                 }
1047         }
1048
1049         for_each_uabi_engine(engine, i915) {
1050                 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
1051                         if (!engine_event_status(engine,
1052                                                  engine_events[i].sample))
1053                                 count++;
1054                 }
1055         }
1056
1057         /* Allocate attribute objects and table. */
1058         i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL);
1059         if (!i915_attr)
1060                 goto err_alloc;
1061
1062         pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL);
1063         if (!pmu_attr)
1064                 goto err_alloc;
1065
1066         /* Max one pointer of each attribute type plus a termination entry. */
1067         attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL);
1068         if (!attr)
1069                 goto err_alloc;
1070
1071         i915_iter = i915_attr;
1072         pmu_iter = pmu_attr;
1073         attr_iter = attr;
1074
1075         /* Initialize supported non-engine counters. */
1076         for_each_gt(gt, i915, j) {
1077                 for (i = 0; i < ARRAY_SIZE(events); i++) {
1078                         u64 config = ___I915_PMU_OTHER(j, events[i].counter);
1079                         char *str;
1080
1081                         if (config_status(i915, config))
1082                                 continue;
1083
1084                         if (events[i].global || !HAS_EXTRA_GT_LIST(i915))
1085                                 str = kstrdup(events[i].name, GFP_KERNEL);
1086                         else
1087                                 str = kasprintf(GFP_KERNEL, "%s-gt%u",
1088                                                 events[i].name, j);
1089                         if (!str)
1090                                 goto err;
1091
1092                         *attr_iter++ = &i915_iter->attr.attr;
1093                         i915_iter = add_i915_attr(i915_iter, str, config);
1094
1095                         if (events[i].unit) {
1096                                 if (events[i].global || !HAS_EXTRA_GT_LIST(i915))
1097                                         str = kasprintf(GFP_KERNEL, "%s.unit",
1098                                                         events[i].name);
1099                                 else
1100                                         str = kasprintf(GFP_KERNEL, "%s-gt%u.unit",
1101                                                         events[i].name, j);
1102                                 if (!str)
1103                                         goto err;
1104
1105                                 *attr_iter++ = &pmu_iter->attr.attr;
1106                                 pmu_iter = add_pmu_attr(pmu_iter, str,
1107                                                         events[i].unit);
1108                         }
1109                 }
1110         }
1111
1112         /* Initialize supported engine counters. */
1113         for_each_uabi_engine(engine, i915) {
1114                 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
1115                         char *str;
1116
1117                         if (engine_event_status(engine,
1118                                                 engine_events[i].sample))
1119                                 continue;
1120
1121                         str = kasprintf(GFP_KERNEL, "%s-%s",
1122                                         engine->name, engine_events[i].name);
1123                         if (!str)
1124                                 goto err;
1125
1126                         *attr_iter++ = &i915_iter->attr.attr;
1127                         i915_iter =
1128                                 add_i915_attr(i915_iter, str,
1129                                               __I915_PMU_ENGINE(engine->uabi_class,
1130                                                                 engine->uabi_instance,
1131                                                                 engine_events[i].sample));
1132
1133                         str = kasprintf(GFP_KERNEL, "%s-%s.unit",
1134                                         engine->name, engine_events[i].name);
1135                         if (!str)
1136                                 goto err;
1137
1138                         *attr_iter++ = &pmu_iter->attr.attr;
1139                         pmu_iter = add_pmu_attr(pmu_iter, str, "ns");
1140                 }
1141         }
1142
1143         pmu->i915_attr = i915_attr;
1144         pmu->pmu_attr = pmu_attr;
1145
1146         return attr;
1147
1148 err:;
1149         for (attr_iter = attr; *attr_iter; attr_iter++)
1150                 kfree((*attr_iter)->name);
1151
1152 err_alloc:
1153         kfree(attr);
1154         kfree(i915_attr);
1155         kfree(pmu_attr);
1156
1157         return NULL;
1158 }
1159
1160 static void free_event_attributes(struct i915_pmu *pmu)
1161 {
1162         struct attribute **attr_iter = pmu->events_attr_group.attrs;
1163
1164         for (; *attr_iter; attr_iter++)
1165                 kfree((*attr_iter)->name);
1166
1167         kfree(pmu->events_attr_group.attrs);
1168         kfree(pmu->i915_attr);
1169         kfree(pmu->pmu_attr);
1170
1171         pmu->events_attr_group.attrs = NULL;
1172         pmu->i915_attr = NULL;
1173         pmu->pmu_attr = NULL;
1174 }
1175
1176 static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
1177 {
1178         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node);
1179
1180         /* Select the first online CPU as a designated reader. */
1181         if (cpumask_empty(&i915_pmu_cpumask))
1182                 cpumask_set_cpu(cpu, &i915_pmu_cpumask);
1183
1184         return 0;
1185 }
1186
1187 static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
1188 {
1189         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node);
1190         unsigned int target = i915_pmu_target_cpu;
1191
1192         /*
1193          * Unregistering an instance generates a CPU offline event which we must
1194          * ignore to avoid incorrectly modifying the shared i915_pmu_cpumask.
1195          */
1196         if (!pmu->registered)
1197                 return 0;
1198
1199         if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) {
1200                 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
1201
1202                 /* Migrate events if there is a valid target */
1203                 if (target < nr_cpu_ids) {
1204                         cpumask_set_cpu(target, &i915_pmu_cpumask);
1205                         i915_pmu_target_cpu = target;
1206                 }
1207         }
1208
1209         if (target < nr_cpu_ids && target != pmu->cpuhp.cpu) {
1210                 perf_pmu_migrate_context(&pmu->base, cpu, target);
1211                 pmu->cpuhp.cpu = target;
1212         }
1213
1214         return 0;
1215 }
1216
1217 static enum cpuhp_state cpuhp_state = CPUHP_INVALID;
1218
1219 int i915_pmu_init(void)
1220 {
1221         int ret;
1222
1223         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1224                                       "perf/x86/intel/i915:online",
1225                                       i915_pmu_cpu_online,
1226                                       i915_pmu_cpu_offline);
1227         if (ret < 0)
1228                 pr_notice("Failed to setup cpuhp state for i915 PMU! (%d)\n",
1229                           ret);
1230         else
1231                 cpuhp_state = ret;
1232
1233         return 0;
1234 }
1235
1236 void i915_pmu_exit(void)
1237 {
1238         if (cpuhp_state != CPUHP_INVALID)
1239                 cpuhp_remove_multi_state(cpuhp_state);
1240 }
1241
1242 static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu)
1243 {
1244         if (cpuhp_state == CPUHP_INVALID)
1245                 return -EINVAL;
1246
1247         return cpuhp_state_add_instance(cpuhp_state, &pmu->cpuhp.node);
1248 }
1249
1250 static void i915_pmu_unregister_cpuhp_state(struct i915_pmu *pmu)
1251 {
1252         cpuhp_state_remove_instance(cpuhp_state, &pmu->cpuhp.node);
1253 }
1254
1255 void i915_pmu_register(struct drm_i915_private *i915)
1256 {
1257         struct i915_pmu *pmu = &i915->pmu;
1258         const struct attribute_group *attr_groups[] = {
1259                 &i915_pmu_format_attr_group,
1260                 &pmu->events_attr_group,
1261                 &i915_pmu_cpumask_attr_group,
1262                 NULL
1263         };
1264         int ret = -ENOMEM;
1265
1266         spin_lock_init(&pmu->lock);
1267         hrtimer_init(&pmu->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1268         pmu->timer.function = i915_sample;
1269         pmu->cpuhp.cpu = -1;
1270         init_rc6(pmu);
1271
1272         if (IS_DGFX(i915)) {
1273                 pmu->name = kasprintf(GFP_KERNEL,
1274                                       "i915_%s",
1275                                       dev_name(i915->drm.dev));
1276                 if (pmu->name) {
1277                         /* tools/perf reserves colons as special. */
1278                         strreplace((char *)pmu->name, ':', '_');
1279                 }
1280         } else {
1281                 pmu->name = "i915";
1282         }
1283         if (!pmu->name)
1284                 goto err;
1285
1286         pmu->events_attr_group.name = "events";
1287         pmu->events_attr_group.attrs = create_event_attributes(pmu);
1288         if (!pmu->events_attr_group.attrs)
1289                 goto err_name;
1290
1291         pmu->base.attr_groups = kmemdup(attr_groups, sizeof(attr_groups),
1292                                         GFP_KERNEL);
1293         if (!pmu->base.attr_groups)
1294                 goto err_attr;
1295
1296         pmu->base.module        = THIS_MODULE;
1297         pmu->base.task_ctx_nr   = perf_invalid_context;
1298         pmu->base.event_init    = i915_pmu_event_init;
1299         pmu->base.add           = i915_pmu_event_add;
1300         pmu->base.del           = i915_pmu_event_del;
1301         pmu->base.start         = i915_pmu_event_start;
1302         pmu->base.stop          = i915_pmu_event_stop;
1303         pmu->base.read          = i915_pmu_event_read;
1304         pmu->base.event_idx     = i915_pmu_event_event_idx;
1305
1306         ret = perf_pmu_register(&pmu->base, pmu->name, -1);
1307         if (ret)
1308                 goto err_groups;
1309
1310         ret = i915_pmu_register_cpuhp_state(pmu);
1311         if (ret)
1312                 goto err_unreg;
1313
1314         pmu->registered = true;
1315
1316         return;
1317
1318 err_unreg:
1319         perf_pmu_unregister(&pmu->base);
1320 err_groups:
1321         kfree(pmu->base.attr_groups);
1322 err_attr:
1323         free_event_attributes(pmu);
1324 err_name:
1325         if (IS_DGFX(i915))
1326                 kfree(pmu->name);
1327 err:
1328         drm_notice(&i915->drm, "Failed to register PMU!\n");
1329 }
1330
1331 void i915_pmu_unregister(struct drm_i915_private *i915)
1332 {
1333         struct i915_pmu *pmu = &i915->pmu;
1334
1335         if (!pmu->registered)
1336                 return;
1337
1338         /* Disconnect the PMU callbacks */
1339         pmu->registered = false;
1340
1341         hrtimer_cancel(&pmu->timer);
1342
1343         i915_pmu_unregister_cpuhp_state(pmu);
1344
1345         perf_pmu_unregister(&pmu->base);
1346         kfree(pmu->base.attr_groups);
1347         if (IS_DGFX(i915))
1348                 kfree(pmu->name);
1349         free_event_attributes(pmu);
1350 }