drivers/gpu/drm/i915/i915_pmu.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/perf_event.h>
  26 #include <linux/pm_runtime.h>
  27
  28 #include "i915_drv.h"
  29 #include "i915_pmu.h"
  30 #include "intel_ringbuffer.h"
  31
  32 /* Frequency for the sampling timer for events which need it. */
  33 #define FREQUENCY 200
  34 #define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)
  35
  36 #define ENGINE_SAMPLE_MASK \
  37         (BIT(I915_SAMPLE_BUSY) | \
  38          BIT(I915_SAMPLE_WAIT) | \
  39          BIT(I915_SAMPLE_SEMA))
  40
  41 #define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS)
  42
  43 static cpumask_t i915_pmu_cpumask;
  44
  45 static u8 engine_config_sample(u64 config)
  46 {
  47         return config & I915_PMU_SAMPLE_MASK;
  48 }
  49
  50 static u8 engine_event_sample(struct perf_event *event)
  51 {
  52         return engine_config_sample(event->attr.config);
  53 }
  54
  55 static u8 engine_event_class(struct perf_event *event)
  56 {
  57         return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
  58 }
  59
  60 static u8 engine_event_instance(struct perf_event *event)
  61 {
  62         return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
  63 }
  64
  65 static bool is_engine_config(u64 config)
  66 {
  67         return config < __I915_PMU_OTHER(0);
  68 }
  69
  70 static unsigned int config_enabled_bit(u64 config)
  71 {
  72         if (is_engine_config(config))
  73                 return engine_config_sample(config);
  74         else
  75                 return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0));
  76 }
  77
  78 static u64 config_enabled_mask(u64 config)
  79 {
  80         return BIT_ULL(config_enabled_bit(config));
  81 }
  82
  83 static bool is_engine_event(struct perf_event *event)
  84 {
  85         return is_engine_config(event->attr.config);
  86 }
  87
  88 static unsigned int event_enabled_bit(struct perf_event *event)
  89 {
  90         return config_enabled_bit(event->attr.config);
  91 }
  92
  93 static bool pmu_needs_timer(struct drm_i915_private *i915, bool gpu_active)
  94 {
  95         u64 enable;
  96
  97         /*
  98          * Only some counters need the sampling timer.
  99          *
 100          * We start with a bitmask of all currently enabled events.
 101          */
 102         enable = i915->pmu.enable;
 103
 104         /*
 105          * Mask out all the ones which do not need the timer, or in
 106          * other words keep all the ones that could need the timer.
 107          */
 108         enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) |
 109                   config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) |
 110                   ENGINE_SAMPLE_MASK;
 111
 112         /*
 113          * When the GPU is idle per-engine counters do not need to be
 114          * running so clear those bits out.
 115          */
 116         if (!gpu_active)
 117                 enable &= ~ENGINE_SAMPLE_MASK;
 118         /*
 119          * Also there is software busyness tracking available we do not
 120          * need the timer for I915_SAMPLE_BUSY counter.
 121          *
 122          * Use RCS as proxy for all engines.
 123          */
 124         else if (intel_engine_supports_stats(i915->engine[RCS]))
 125                 enable &= ~BIT(I915_SAMPLE_BUSY);
 126
 127         /*
 128          * If some bits remain it means we need the sampling timer running.
 129          */
 130         return enable;
 131 }
 132
 133 void i915_pmu_gt_parked(struct drm_i915_private *i915)
 134 {
 135         if (!i915->pmu.base.event_init)
 136                 return;
 137
 138         spin_lock_irq(&i915->pmu.lock);
 139         /*
 140          * Signal sampling timer to stop if only engine events are enabled and
 141          * GPU went idle.
 142          */
 143         i915->pmu.timer_enabled = pmu_needs_timer(i915, false);
 144         spin_unlock_irq(&i915->pmu.lock);
 145 }
 146
 147 static void __i915_pmu_maybe_start_timer(struct drm_i915_private *i915)
 148 {
 149         if (!i915->pmu.timer_enabled && pmu_needs_timer(i915, true)) {
 150                 i915->pmu.timer_enabled = true;
 151                 hrtimer_start_range_ns(&i915->pmu.timer,
 152                                        ns_to_ktime(PERIOD), 0,
 153                                        HRTIMER_MODE_REL_PINNED);
 154         }
 155 }
 156
 157 void i915_pmu_gt_unparked(struct drm_i915_private *i915)
 158 {
 159         if (!i915->pmu.base.event_init)
 160                 return;
 161
 162         spin_lock_irq(&i915->pmu.lock);
 163         /*
 164          * Re-enable sampling timer when GPU goes active.
 165          */
 166         __i915_pmu_maybe_start_timer(i915);
 167         spin_unlock_irq(&i915->pmu.lock);
 168 }
 169
 170 static bool grab_forcewake(struct drm_i915_private *i915, bool fw)
 171 {
 172         if (!fw)
 173                 intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
 174
 175         return true;
 176 }
 177
 178 static void
 179 update_sample(struct i915_pmu_sample *sample, u32 unit, u32 val)
 180 {
 181         sample->cur += mul_u32_u32(val, unit);
 182 }
 183
 184 static void engines_sample(struct drm_i915_private *dev_priv)
 185 {
 186         struct intel_engine_cs *engine;
 187         enum intel_engine_id id;
 188         bool fw = false;
 189
 190         if ((dev_priv->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
 191                 return;
 192
 193         if (!dev_priv->gt.awake)
 194                 return;
 195
 196         if (!intel_runtime_pm_get_if_in_use(dev_priv))
 197                 return;
 198
 199         for_each_engine(engine, dev_priv, id) {
 200                 u32 current_seqno = intel_engine_get_seqno(engine);
 201                 u32 last_seqno = intel_engine_last_submit(engine);
 202                 u32 val;
 203
 204                 val = !i915_seqno_passed(current_seqno, last_seqno);
 205
 206                 update_sample(&engine->pmu.sample[I915_SAMPLE_BUSY],
 207                               PERIOD, val);
 208
 209                 if (val && (engine->pmu.enable &
 210                     (BIT(I915_SAMPLE_WAIT) | BIT(I915_SAMPLE_SEMA)))) {
 211                         fw = grab_forcewake(dev_priv, fw);
 212
 213                         val = I915_READ_FW(RING_CTL(engine->mmio_base));
 214                 } else {
 215                         val = 0;
 216                 }
 217
 218                 update_sample(&engine->pmu.sample[I915_SAMPLE_WAIT],
 219                               PERIOD, !!(val & RING_WAIT));
 220
 221                 update_sample(&engine->pmu.sample[I915_SAMPLE_SEMA],
 222                               PERIOD, !!(val & RING_WAIT_SEMAPHORE));
 223         }
 224
 225         if (fw)
 226                 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 227
 228         intel_runtime_pm_put(dev_priv);
 229 }
 230
 231 static void frequency_sample(struct drm_i915_private *dev_priv)
 232 {
 233         if (dev_priv->pmu.enable &
 234             config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) {
 235                 u32 val;
 236
 237                 val = dev_priv->gt_pm.rps.cur_freq;
 238                 if (dev_priv->gt.awake &&
 239                     intel_runtime_pm_get_if_in_use(dev_priv)) {
 240                         val = intel_get_cagf(dev_priv,
 241                                              I915_READ_NOTRACE(GEN6_RPSTAT1));
 242                         intel_runtime_pm_put(dev_priv);
 243                 }
 244
 245                 update_sample(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_ACT],
 246                               1, intel_gpu_freq(dev_priv, val));
 247         }
 248
 249         if (dev_priv->pmu.enable &
 250             config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) {
 251                 update_sample(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_REQ], 1,
 252                               intel_gpu_freq(dev_priv,
 253                                              dev_priv->gt_pm.rps.cur_freq));
 254         }
 255 }
 256
 257 static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer)
 258 {
 259         struct drm_i915_private *i915 =
 260                 container_of(hrtimer, struct drm_i915_private, pmu.timer);
 261
 262         if (!READ_ONCE(i915->pmu.timer_enabled))
 263                 return HRTIMER_NORESTART;
 264
 265         engines_sample(i915);
 266         frequency_sample(i915);
 267
 268         hrtimer_forward_now(hrtimer, ns_to_ktime(PERIOD));
 269         return HRTIMER_RESTART;
 270 }
 271
 272 static u64 count_interrupts(struct drm_i915_private *i915)
 273 {
 274         /* open-coded kstat_irqs() */
 275         struct irq_desc *desc = irq_to_desc(i915->drm.pdev->irq);
 276         u64 sum = 0;
 277         int cpu;
 278
 279         if (!desc || !desc->kstat_irqs)
 280                 return 0;
 281
 282         for_each_possible_cpu(cpu)
 283                 sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
 284
 285         return sum;
 286 }
 287
 288 static void engine_event_destroy(struct perf_event *event)
 289 {
 290         struct drm_i915_private *i915 =
 291                 container_of(event->pmu, typeof(*i915), pmu.base);
 292         struct intel_engine_cs *engine;
 293
 294         engine = intel_engine_lookup_user(i915,
 295                                           engine_event_class(event),
 296                                           engine_event_instance(event));
 297         if (WARN_ON_ONCE(!engine))
 298                 return;
 299
 300         if (engine_event_sample(event) == I915_SAMPLE_BUSY &&
 301             intel_engine_supports_stats(engine))
 302                 intel_disable_engine_stats(engine);
 303 }
 304
 305 static void i915_pmu_event_destroy(struct perf_event *event)
 306 {
 307         WARN_ON(event->parent);
 308
 309         if (is_engine_event(event))
 310                 engine_event_destroy(event);
 311 }
 312
 313 static int
 314 engine_event_status(struct intel_engine_cs *engine,
 315                     enum drm_i915_pmu_engine_sample sample)
 316 {
 317         switch (sample) {
 318         case I915_SAMPLE_BUSY:
 319         case I915_SAMPLE_WAIT:
 320                 break;
 321         case I915_SAMPLE_SEMA:
 322                 if (INTEL_GEN(engine->i915) < 6)
 323                         return -ENODEV;
 324                 break;
 325         default:
 326                 return -ENOENT;
 327         }
 328
 329         return 0;
 330 }
 331
 332 static int
 333 config_status(struct drm_i915_private *i915, u64 config)
 334 {
 335         switch (config) {
 336         case I915_PMU_ACTUAL_FREQUENCY:
 337                 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 338                         /* Requires a mutex for sampling! */
 339                         return -ENODEV;
 340                 /* Fall-through. */
 341         case I915_PMU_REQUESTED_FREQUENCY:
 342                 if (INTEL_GEN(i915) < 6)
 343                         return -ENODEV;
 344                 break;
 345         case I915_PMU_INTERRUPTS:
 346                 break;
 347         case I915_PMU_RC6_RESIDENCY:
 348                 if (!HAS_RC6(i915))
 349                         return -ENODEV;
 350                 break;
 351         default:
 352                 return -ENOENT;
 353         }
 354
 355         return 0;
 356 }
 357
 358 static int engine_event_init(struct perf_event *event)
 359 {
 360         struct drm_i915_private *i915 =
 361                 container_of(event->pmu, typeof(*i915), pmu.base);
 362         struct intel_engine_cs *engine;
 363         u8 sample;
 364         int ret;
 365
 366         engine = intel_engine_lookup_user(i915, engine_event_class(event),
 367                                           engine_event_instance(event));
 368         if (!engine)
 369                 return -ENODEV;
 370
 371         sample = engine_event_sample(event);
 372         ret = engine_event_status(engine, sample);
 373         if (ret)
 374                 return ret;
 375
 376         if (sample == I915_SAMPLE_BUSY && intel_engine_supports_stats(engine))
 377                 ret = intel_enable_engine_stats(engine);
 378
 379         return ret;
 380 }
 381
 382 static int i915_pmu_event_init(struct perf_event *event)
 383 {
 384         struct drm_i915_private *i915 =
 385                 container_of(event->pmu, typeof(*i915), pmu.base);
 386         int ret;
 387
 388         if (event->attr.type != event->pmu->type)
 389                 return -ENOENT;
 390
 391         /* unsupported modes and filters */
 392         if (event->attr.sample_period) /* no sampling */
 393                 return -EINVAL;
 394
 395         if (has_branch_stack(event))
 396                 return -EOPNOTSUPP;
 397
 398         if (event->cpu < 0)
 399                 return -EINVAL;
 400
 401         /* only allow running on one cpu at a time */
 402         if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask))
 403                 return -EINVAL;
 404
 405         if (is_engine_event(event))
 406                 ret = engine_event_init(event);
 407         else
 408                 ret = config_status(i915, event->attr.config);
 409         if (ret)
 410                 return ret;
 411
 412         if (!event->parent)
 413                 event->destroy = i915_pmu_event_destroy;
 414
 415         return 0;
 416 }
 417
 418 static u64 __get_rc6(struct drm_i915_private *i915)
 419 {
 420         u64 val;
 421
 422         val = intel_rc6_residency_ns(i915,
 423                                      IS_VALLEYVIEW(i915) ?
 424                                      VLV_GT_RENDER_RC6 :
 425                                      GEN6_GT_GFX_RC6);
 426
 427         if (HAS_RC6p(i915))
 428                 val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6p);
 429
 430         if (HAS_RC6pp(i915))
 431                 val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6pp);
 432
 433         return val;
 434 }
 435
 436 static u64 get_rc6(struct drm_i915_private *i915)
 437 {
 438 #if IS_ENABLED(CONFIG_PM)
 439         unsigned long flags;
 440         u64 val;
 441
 442         if (intel_runtime_pm_get_if_in_use(i915)) {
 443                 val = __get_rc6(i915);
 444                 intel_runtime_pm_put(i915);
 445
 446                 /*
 447                  * If we are coming back from being runtime suspended we must
 448                  * be careful not to report a larger value than returned
 449                  * previously.
 450                  */
 451
 452                 spin_lock_irqsave(&i915->pmu.lock, flags);
 453
 454                 if (val >= i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) {
 455                         i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = 0;
 456                         i915->pmu.sample[__I915_SAMPLE_RC6].cur = val;
 457                 } else {
 458                         val = i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur;
 459                 }
 460
 461                 spin_unlock_irqrestore(&i915->pmu.lock, flags);
 462         } else {
 463                 struct pci_dev *pdev = i915->drm.pdev;
 464                 struct device *kdev = &pdev->dev;
 465
 466                 /*
 467                  * We are runtime suspended.
 468                  *
 469                  * Report the delta from when the device was suspended to now,
 470                  * on top of the last known real value, as the approximated RC6
 471                  * counter value.
 472                  */
 473                 spin_lock_irqsave(&i915->pmu.lock, flags);
 474                 spin_lock(&kdev->power.lock);
 475
 476                 if (!i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur)
 477                         i915->pmu.suspended_jiffies_last =
 478                                                 kdev->power.suspended_jiffies;
 479
 480                 val = kdev->power.suspended_jiffies -
 481                       i915->pmu.suspended_jiffies_last;
 482                 val += jiffies - kdev->power.accounting_timestamp;
 483
 484                 spin_unlock(&kdev->power.lock);
 485
 486                 val = jiffies_to_nsecs(val);
 487                 val += i915->pmu.sample[__I915_SAMPLE_RC6].cur;
 488                 i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = val;
 489
 490                 spin_unlock_irqrestore(&i915->pmu.lock, flags);
 491         }
 492
 493         return val;
 494 #else
 495         return __get_rc6(i915);
 496 #endif
 497 }
 498
 499 static u64 __i915_pmu_event_read(struct perf_event *event)
 500 {
 501         struct drm_i915_private *i915 =
 502                 container_of(event->pmu, typeof(*i915), pmu.base);
 503         u64 val = 0;
 504
 505         if (is_engine_event(event)) {
 506                 u8 sample = engine_event_sample(event);
 507                 struct intel_engine_cs *engine;
 508
 509                 engine = intel_engine_lookup_user(i915,
 510                                                   engine_event_class(event),
 511                                                   engine_event_instance(event));
 512
 513                 if (WARN_ON_ONCE(!engine)) {
 514                         /* Do nothing */
 515                 } else if (sample == I915_SAMPLE_BUSY &&
 516                            intel_engine_supports_stats(engine)) {
 517                         val = ktime_to_ns(intel_engine_get_busy_time(engine));
 518                 } else {
 519                         val = engine->pmu.sample[sample].cur;
 520                 }
 521         } else {
 522                 switch (event->attr.config) {
 523                 case I915_PMU_ACTUAL_FREQUENCY:
 524                         val =
 525                            div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_ACT].cur,
 526                                    FREQUENCY);
 527                         break;
 528                 case I915_PMU_REQUESTED_FREQUENCY:
 529                         val =
 530                            div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_REQ].cur,
 531                                    FREQUENCY);
 532                         break;
 533                 case I915_PMU_INTERRUPTS:
 534                         val = count_interrupts(i915);
 535                         break;
 536                 case I915_PMU_RC6_RESIDENCY:
 537                         val = get_rc6(i915);
 538                         break;
 539                 }
 540         }
 541
 542         return val;
 543 }
 544
 545 static void i915_pmu_event_read(struct perf_event *event)
 546 {
 547         struct hw_perf_event *hwc = &event->hw;
 548         u64 prev, new;
 549
 550 again:
 551         prev = local64_read(&hwc->prev_count);
 552         new = __i915_pmu_event_read(event);
 553
 554         if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev)
 555                 goto again;
 556
 557         local64_add(new - prev, &event->count);
 558 }
 559
 560 static void i915_pmu_enable(struct perf_event *event)
 561 {
 562         struct drm_i915_private *i915 =
 563                 container_of(event->pmu, typeof(*i915), pmu.base);
 564         unsigned int bit = event_enabled_bit(event);
 565         unsigned long flags;
 566
 567         spin_lock_irqsave(&i915->pmu.lock, flags);
 568
 569         /*
 570          * Update the bitmask of enabled events and increment
 571          * the event reference counter.
 572          */
 573         GEM_BUG_ON(bit >= I915_PMU_MASK_BITS);
 574         GEM_BUG_ON(i915->pmu.enable_count[bit] == ~0);
 575         i915->pmu.enable |= BIT_ULL(bit);
 576         i915->pmu.enable_count[bit]++;
 577
 578         /*
 579          * Start the sampling timer if needed and not already enabled.
 580          */
 581         __i915_pmu_maybe_start_timer(i915);
 582
 583         /*
 584          * For per-engine events the bitmask and reference counting
 585          * is stored per engine.
 586          */
 587         if (is_engine_event(event)) {
 588                 u8 sample = engine_event_sample(event);
 589                 struct intel_engine_cs *engine;
 590
 591                 engine = intel_engine_lookup_user(i915,
 592                                                   engine_event_class(event),
 593                                                   engine_event_instance(event));
 594                 GEM_BUG_ON(!engine);
 595                 engine->pmu.enable |= BIT(sample);
 596
 597                 GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS);
 598                 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);
 599                 engine->pmu.enable_count[sample]++;
 600         }
 601
 602         spin_unlock_irqrestore(&i915->pmu.lock, flags);
 603
 604         /*
 605          * Store the current counter value so we can report the correct delta
 606          * for all listeners. Even when the event was already enabled and has
 607          * an existing non-zero value.
 608          */
 609         local64_set(&event->hw.prev_count, __i915_pmu_event_read(event));
 610 }
 611
 612 static void i915_pmu_disable(struct perf_event *event)
 613 {
 614         struct drm_i915_private *i915 =
 615                 container_of(event->pmu, typeof(*i915), pmu.base);
 616         unsigned int bit = event_enabled_bit(event);
 617         unsigned long flags;
 618
 619         spin_lock_irqsave(&i915->pmu.lock, flags);
 620
 621         if (is_engine_event(event)) {
 622                 u8 sample = engine_event_sample(event);
 623                 struct intel_engine_cs *engine;
 624
 625                 engine = intel_engine_lookup_user(i915,
 626                                                   engine_event_class(event),
 627                                                   engine_event_instance(event));
 628                 GEM_BUG_ON(!engine);
 629                 GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS);
 630                 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);
 631                 /*
 632                  * Decrement the reference count and clear the enabled
 633                  * bitmask when the last listener on an event goes away.
 634                  */
 635                 if (--engine->pmu.enable_count[sample] == 0)
 636                         engine->pmu.enable &= ~BIT(sample);
 637         }
 638
 639         GEM_BUG_ON(bit >= I915_PMU_MASK_BITS);
 640         GEM_BUG_ON(i915->pmu.enable_count[bit] == 0);
 641         /*
 642          * Decrement the reference count and clear the enabled
 643          * bitmask when the last listener on an event goes away.
 644          */
 645         if (--i915->pmu.enable_count[bit] == 0) {
 646                 i915->pmu.enable &= ~BIT_ULL(bit);
 647                 i915->pmu.timer_enabled &= pmu_needs_timer(i915, true);
 648         }
 649
 650         spin_unlock_irqrestore(&i915->pmu.lock, flags);
 651 }
 652
 653 static void i915_pmu_event_start(struct perf_event *event, int flags)
 654 {
 655         i915_pmu_enable(event);
 656         event->hw.state = 0;
 657 }
 658
 659 static void i915_pmu_event_stop(struct perf_event *event, int flags)
 660 {
 661         if (flags & PERF_EF_UPDATE)
 662                 i915_pmu_event_read(event);
 663         i915_pmu_disable(event);
 664         event->hw.state = PERF_HES_STOPPED;
 665 }
 666
 667 static int i915_pmu_event_add(struct perf_event *event, int flags)
 668 {
 669         if (flags & PERF_EF_START)
 670                 i915_pmu_event_start(event, flags);
 671
 672         return 0;
 673 }
 674
 675 static void i915_pmu_event_del(struct perf_event *event, int flags)
 676 {
 677         i915_pmu_event_stop(event, PERF_EF_UPDATE);
 678 }
 679
 680 static int i915_pmu_event_event_idx(struct perf_event *event)
 681 {
 682         return 0;
 683 }
 684
 685 struct i915_str_attribute {
 686         struct device_attribute attr;
 687         const char *str;
 688 };
 689
 690 static ssize_t i915_pmu_format_show(struct device *dev,
 691                                     struct device_attribute *attr, char *buf)
 692 {
 693         struct i915_str_attribute *eattr;
 694
 695         eattr = container_of(attr, struct i915_str_attribute, attr);
 696         return sprintf(buf, "%s\n", eattr->str);
 697 }
 698
 699 #define I915_PMU_FORMAT_ATTR(_name, _config) \
 700         (&((struct i915_str_attribute[]) { \
 701                 { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \
 702                   .str = _config, } \
 703         })[0].attr.attr)
 704
 705 static struct attribute *i915_pmu_format_attrs[] = {
 706         I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
 707         NULL,
 708 };
 709
 710 static const struct attribute_group i915_pmu_format_attr_group = {
 711         .name = "format",
 712         .attrs = i915_pmu_format_attrs,
 713 };
 714
 715 struct i915_ext_attribute {
 716         struct device_attribute attr;
 717         unsigned long val;
 718 };
 719
 720 static ssize_t i915_pmu_event_show(struct device *dev,
 721                                    struct device_attribute *attr, char *buf)
 722 {
 723         struct i915_ext_attribute *eattr;
 724
 725         eattr = container_of(attr, struct i915_ext_attribute, attr);
 726         return sprintf(buf, "config=0x%lx\n", eattr->val);
 727 }
 728
 729 static struct attribute_group i915_pmu_events_attr_group = {
 730         .name = "events",
 731         /* Patch in attrs at runtime. */
 732 };
 733
 734 static ssize_t
 735 i915_pmu_get_attr_cpumask(struct device *dev,
 736                           struct device_attribute *attr,
 737                           char *buf)
 738 {
 739         return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask);
 740 }
 741
 742 static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL);
 743
 744 static struct attribute *i915_cpumask_attrs[] = {
 745         &dev_attr_cpumask.attr,
 746         NULL,
 747 };
 748
 749 static const struct attribute_group i915_pmu_cpumask_attr_group = {
 750         .attrs = i915_cpumask_attrs,
 751 };
 752
 753 static const struct attribute_group *i915_pmu_attr_groups[] = {
 754         &i915_pmu_format_attr_group,
 755         &i915_pmu_events_attr_group,
 756         &i915_pmu_cpumask_attr_group,
 757         NULL
 758 };
 759
 760 #define __event(__config, __name, __unit) \
 761 { \
 762         .config = (__config), \
 763         .name = (__name), \
 764         .unit = (__unit), \
 765 }
 766
 767 #define __engine_event(__sample, __name) \
 768 { \
 769         .sample = (__sample), \
 770         .name = (__name), \
 771 }
 772
 773 static struct i915_ext_attribute *
 774 add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config)
 775 {
 776         sysfs_attr_init(&attr->attr.attr);
 777         attr->attr.attr.name = name;
 778         attr->attr.attr.mode = 0444;
 779         attr->attr.show = i915_pmu_event_show;
 780         attr->val = config;
 781
 782         return ++attr;
 783 }
 784
 785 static struct perf_pmu_events_attr *
 786 add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name,
 787              const char *str)
 788 {
 789         sysfs_attr_init(&attr->attr.attr);
 790         attr->attr.attr.name = name;
 791         attr->attr.attr.mode = 0444;
 792         attr->attr.show = perf_event_sysfs_show;
 793         attr->event_str = str;
 794
 795         return ++attr;
 796 }
 797
 798 static struct attribute **
 799 create_event_attributes(struct drm_i915_private *i915)
 800 {
 801         static const struct {
 802                 u64 config;
 803                 const char *name;
 804                 const char *unit;
 805         } events[] = {
 806                 __event(I915_PMU_ACTUAL_FREQUENCY, "actual-frequency", "MHz"),
 807                 __event(I915_PMU_REQUESTED_FREQUENCY, "requested-frequency", "MHz"),
 808                 __event(I915_PMU_INTERRUPTS, "interrupts", NULL),
 809                 __event(I915_PMU_RC6_RESIDENCY, "rc6-residency", "ns"),
 810         };
 811         static const struct {
 812                 enum drm_i915_pmu_engine_sample sample;
 813                 char *name;
 814         } engine_events[] = {
 815                 __engine_event(I915_SAMPLE_BUSY, "busy"),
 816                 __engine_event(I915_SAMPLE_SEMA, "sema"),
 817                 __engine_event(I915_SAMPLE_WAIT, "wait"),
 818         };
 819         unsigned int count = 0;
 820         struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter;
 821         struct i915_ext_attribute *i915_attr = NULL, *i915_iter;
 822         struct attribute **attr = NULL, **attr_iter;
 823         struct intel_engine_cs *engine;
 824         enum intel_engine_id id;
 825         unsigned int i;
 826
 827         /* Count how many counters we will be exposing. */
 828         for (i = 0; i < ARRAY_SIZE(events); i++) {
 829                 if (!config_status(i915, events[i].config))
 830                         count++;
 831         }
 832
 833         for_each_engine(engine, i915, id) {
 834                 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
 835                         if (!engine_event_status(engine,
 836                                                  engine_events[i].sample))
 837                                 count++;
 838                 }
 839         }
 840
 841         /* Allocate attribute objects and table. */
 842         i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL);
 843         if (!i915_attr)
 844                 goto err_alloc;
 845
 846         pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL);
 847         if (!pmu_attr)
 848                 goto err_alloc;
 849
 850         /* Max one pointer of each attribute type plus a termination entry. */
 851         attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL);
 852         if (!attr)
 853                 goto err_alloc;
 854
 855         i915_iter = i915_attr;
 856         pmu_iter = pmu_attr;
 857         attr_iter = attr;
 858
 859         /* Initialize supported non-engine counters. */
 860         for (i = 0; i < ARRAY_SIZE(events); i++) {
 861                 char *str;
 862
 863                 if (config_status(i915, events[i].config))
 864                         continue;
 865
 866                 str = kstrdup(events[i].name, GFP_KERNEL);
 867                 if (!str)
 868                         goto err;
 869
 870                 *attr_iter++ = &i915_iter->attr.attr;
 871                 i915_iter = add_i915_attr(i915_iter, str, events[i].config);
 872
 873                 if (events[i].unit) {
 874                         str = kasprintf(GFP_KERNEL, "%s.unit", events[i].name);
 875                         if (!str)
 876                                 goto err;
 877
 878                         *attr_iter++ = &pmu_iter->attr.attr;
 879                         pmu_iter = add_pmu_attr(pmu_iter, str, events[i].unit);
 880                 }
 881         }
 882
 883         /* Initialize supported engine counters. */
 884         for_each_engine(engine, i915, id) {
 885                 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
 886                         char *str;
 887
 888                         if (engine_event_status(engine,
 889                                                 engine_events[i].sample))
 890                                 continue;
 891
 892                         str = kasprintf(GFP_KERNEL, "%s-%s",
 893                                         engine->name, engine_events[i].name);
 894                         if (!str)
 895                                 goto err;
 896
 897                         *attr_iter++ = &i915_iter->attr.attr;
 898                         i915_iter =
 899                                 add_i915_attr(i915_iter, str,
 900                                               __I915_PMU_ENGINE(engine->uabi_class,
 901                                                                 engine->instance,
 902                                                                 engine_events[i].sample));
 903
 904                         str = kasprintf(GFP_KERNEL, "%s-%s.unit",
 905                                         engine->name, engine_events[i].name);
 906                         if (!str)
 907                                 goto err;
 908
 909                         *attr_iter++ = &pmu_iter->attr.attr;
 910                         pmu_iter = add_pmu_attr(pmu_iter, str, "ns");
 911                 }
 912         }
 913
 914         i915->pmu.i915_attr = i915_attr;
 915         i915->pmu.pmu_attr = pmu_attr;
 916
 917         return attr;
 918
 919 err:;
 920         for (attr_iter = attr; *attr_iter; attr_iter++)
 921                 kfree((*attr_iter)->name);
 922
 923 err_alloc:
 924         kfree(attr);
 925         kfree(i915_attr);
 926         kfree(pmu_attr);
 927
 928         return NULL;
 929 }
 930
 931 static void free_event_attributes(struct drm_i915_private *i915)
 932 {
 933         struct attribute **attr_iter = i915_pmu_events_attr_group.attrs;
 934
 935         for (; *attr_iter; attr_iter++)
 936                 kfree((*attr_iter)->name);
 937
 938         kfree(i915_pmu_events_attr_group.attrs);
 939         kfree(i915->pmu.i915_attr);
 940         kfree(i915->pmu.pmu_attr);
 941
 942         i915_pmu_events_attr_group.attrs = NULL;
 943         i915->pmu.i915_attr = NULL;
 944         i915->pmu.pmu_attr = NULL;
 945 }
 946
 947 static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
 948 {
 949         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
 950
 951         GEM_BUG_ON(!pmu->base.event_init);
 952
 953         /* Select the first online CPU as a designated reader. */
 954         if (!cpumask_weight(&i915_pmu_cpumask))
 955                 cpumask_set_cpu(cpu, &i915_pmu_cpumask);
 956
 957         return 0;
 958 }
 959
 960 static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
 961 {
 962         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
 963         unsigned int target;
 964
 965         GEM_BUG_ON(!pmu->base.event_init);
 966
 967         if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) {
 968                 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
 969                 /* Migrate events if there is a valid target */
 970                 if (target < nr_cpu_ids) {
 971                         cpumask_set_cpu(target, &i915_pmu_cpumask);
 972                         perf_pmu_migrate_context(&pmu->base, cpu, target);
 973                 }
 974         }
 975
 976         return 0;
 977 }
 978
 979 static enum cpuhp_state cpuhp_slot = CPUHP_INVALID;
 980
 981 static int i915_pmu_register_cpuhp_state(struct drm_i915_private *i915)
 982 {
 983         enum cpuhp_state slot;
 984         int ret;
 985
 986         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 987                                       "perf/x86/intel/i915:online",
 988                                       i915_pmu_cpu_online,
 989                                       i915_pmu_cpu_offline);
 990         if (ret < 0)
 991                 return ret;
 992
 993         slot = ret;
 994         ret = cpuhp_state_add_instance(slot, &i915->pmu.node);
 995         if (ret) {
 996                 cpuhp_remove_multi_state(slot);
 997                 return ret;
 998         }
 999
1000         cpuhp_slot = slot;
1001         return 0;
1002 }
1003
1004 static void i915_pmu_unregister_cpuhp_state(struct drm_i915_private *i915)
1005 {
1006         WARN_ON(cpuhp_slot == CPUHP_INVALID);
1007         WARN_ON(cpuhp_state_remove_instance(cpuhp_slot, &i915->pmu.node));
1008         cpuhp_remove_multi_state(cpuhp_slot);
1009 }
1010
1011 void i915_pmu_register(struct drm_i915_private *i915)
1012 {
1013         int ret;
1014
1015         if (INTEL_GEN(i915) <= 2) {
1016                 DRM_INFO("PMU not supported for this GPU.");
1017                 return;
1018         }
1019
1020         i915_pmu_events_attr_group.attrs = create_event_attributes(i915);
1021         if (!i915_pmu_events_attr_group.attrs) {
1022                 ret = -ENOMEM;
1023                 goto err;
1024         }
1025
1026         i915->pmu.base.attr_groups      = i915_pmu_attr_groups;
1027         i915->pmu.base.task_ctx_nr      = perf_invalid_context;
1028         i915->pmu.base.event_init       = i915_pmu_event_init;
1029         i915->pmu.base.add              = i915_pmu_event_add;
1030         i915->pmu.base.del              = i915_pmu_event_del;
1031         i915->pmu.base.start            = i915_pmu_event_start;
1032         i915->pmu.base.stop             = i915_pmu_event_stop;
1033         i915->pmu.base.read             = i915_pmu_event_read;
1034         i915->pmu.base.event_idx        = i915_pmu_event_event_idx;
1035
1036         spin_lock_init(&i915->pmu.lock);
1037         hrtimer_init(&i915->pmu.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1038         i915->pmu.timer.function = i915_sample;
1039
1040         ret = perf_pmu_register(&i915->pmu.base, "i915", -1);
1041         if (ret)
1042                 goto err;
1043
1044         ret = i915_pmu_register_cpuhp_state(i915);
1045         if (ret)
1046                 goto err_unreg;
1047
1048         return;
1049
1050 err_unreg:
1051         perf_pmu_unregister(&i915->pmu.base);
1052 err:
1053         i915->pmu.base.event_init = NULL;
1054         free_event_attributes(i915);
1055         DRM_NOTE("Failed to register PMU! (err=%d)\n", ret);
1056 }
1057
1058 void i915_pmu_unregister(struct drm_i915_private *i915)
1059 {
1060         if (!i915->pmu.base.event_init)
1061                 return;
1062
1063         WARN_ON(i915->pmu.enable);
1064
1065         hrtimer_cancel(&i915->pmu.timer);
1066
1067         i915_pmu_unregister_cpuhp_state(i915);
1068
1069         perf_pmu_unregister(&i915->pmu.base);
1070         i915->pmu.base.event_init = NULL;
1071         free_event_attributes(i915);
1072 }