drivers/perf/fsl_imx9_ddr_perf.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright 2023 NXP
   3
   4 #include <linux/bitfield.h>
   5 #include <linux/init.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/io.h>
   8 #include <linux/module.h>
   9 #include <linux/of.h>
  10 #include <linux/of_address.h>
  11 #include <linux/of_device.h>
  12 #include <linux/of_irq.h>
  13 #include <linux/perf_event.h>
  14
  15 /* Performance monitor configuration */
  16 #define PMCFG1                          0x00
  17 #define PMCFG1_RD_TRANS_FILT_EN         BIT(31)
  18 #define PMCFG1_WR_TRANS_FILT_EN         BIT(30)
  19 #define PMCFG1_RD_BT_FILT_EN            BIT(29)
  20 #define PMCFG1_ID_MASK                  GENMASK(17, 0)
  21
  22 #define PMCFG2                          0x04
  23 #define PMCFG2_ID                       GENMASK(17, 0)
  24
  25 /* Global control register affects all counters and takes priority over local control registers */
  26 #define PMGC0           0x40
  27 /* Global control register bits */
  28 #define PMGC0_FAC       BIT(31)
  29 #define PMGC0_PMIE      BIT(30)
  30 #define PMGC0_FCECE     BIT(29)
  31
  32 /*
  33  * 64bit counter0 exclusively dedicated to counting cycles
  34  * 32bit counters monitor counter-specific events in addition to counting reference events
  35  */
  36 #define PMLCA(n)        (0x40 + 0x10 + (0x10 * n))
  37 #define PMLCB(n)        (0x40 + 0x14 + (0x10 * n))
  38 #define PMC(n)          (0x40 + 0x18 + (0x10 * n))
  39 /* Local control register bits */
  40 #define PMLCA_FC        BIT(31)
  41 #define PMLCA_CE        BIT(26)
  42 #define PMLCA_EVENT     GENMASK(22, 16)
  43
  44 #define NUM_COUNTERS            11
  45 #define CYCLES_COUNTER          0
  46
  47 #define to_ddr_pmu(p)           container_of(p, struct ddr_pmu, pmu)
  48
  49 #define DDR_PERF_DEV_NAME       "imx9_ddr"
  50 #define DDR_CPUHP_CB_NAME       DDR_PERF_DEV_NAME "_perf_pmu"
  51
  52 static DEFINE_IDA(ddr_ida);
  53
  54 struct imx_ddr_devtype_data {
  55         const char *identifier;         /* system PMU identifier for userspace */
  56 };
  57
  58 struct ddr_pmu {
  59         struct pmu pmu;
  60         void __iomem *base;
  61         unsigned int cpu;
  62         struct hlist_node node;
  63         struct device *dev;
  64         struct perf_event *events[NUM_COUNTERS];
  65         int active_events;
  66         enum cpuhp_state cpuhp_state;
  67         const struct imx_ddr_devtype_data *devtype_data;
  68         int irq;
  69         int id;
  70 };
  71
  72 static const struct imx_ddr_devtype_data imx93_devtype_data = {
  73         .identifier = "imx93",
  74 };
  75
  76 static const struct of_device_id imx_ddr_pmu_dt_ids[] = {
  77         {.compatible = "fsl,imx93-ddr-pmu", .data = &imx93_devtype_data},
  78         { /* sentinel */ }
  79 };
  80 MODULE_DEVICE_TABLE(of, imx_ddr_pmu_dt_ids);
  81
  82 static ssize_t ddr_perf_identifier_show(struct device *dev,
  83                                         struct device_attribute *attr,
  84                                         char *page)
  85 {
  86         struct ddr_pmu *pmu = dev_get_drvdata(dev);
  87
  88         return sysfs_emit(page, "%s\n", pmu->devtype_data->identifier);
  89 }
  90
  91 static struct device_attribute ddr_perf_identifier_attr =
  92         __ATTR(identifier, 0444, ddr_perf_identifier_show, NULL);
  93
  94 static struct attribute *ddr_perf_identifier_attrs[] = {
  95         &ddr_perf_identifier_attr.attr,
  96         NULL,
  97 };
  98
  99 static struct attribute_group ddr_perf_identifier_attr_group = {
 100         .attrs = ddr_perf_identifier_attrs,
 101 };
 102
 103 static ssize_t ddr_perf_cpumask_show(struct device *dev,
 104                                      struct device_attribute *attr, char *buf)
 105 {
 106         struct ddr_pmu *pmu = dev_get_drvdata(dev);
 107
 108         return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
 109 }
 110
 111 static struct device_attribute ddr_perf_cpumask_attr =
 112         __ATTR(cpumask, 0444, ddr_perf_cpumask_show, NULL);
 113
 114 static struct attribute *ddr_perf_cpumask_attrs[] = {
 115         &ddr_perf_cpumask_attr.attr,
 116         NULL,
 117 };
 118
 119 static const struct attribute_group ddr_perf_cpumask_attr_group = {
 120         .attrs = ddr_perf_cpumask_attrs,
 121 };
 122
 123 static ssize_t ddr_pmu_event_show(struct device *dev,
 124                                   struct device_attribute *attr, char *page)
 125 {
 126         struct perf_pmu_events_attr *pmu_attr;
 127
 128         pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
 129         return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
 130 }
 131
 132 #define IMX9_DDR_PMU_EVENT_ATTR(_name, _id)                             \
 133         (&((struct perf_pmu_events_attr[]) {                            \
 134                 { .attr = __ATTR(_name, 0444, ddr_pmu_event_show, NULL),\
 135                   .id = _id, }                                          \
 136         })[0].attr.attr)
 137
 138 static struct attribute *ddr_perf_events_attrs[] = {
 139         /* counter0 cycles event */
 140         IMX9_DDR_PMU_EVENT_ATTR(cycles, 0),
 141
 142         /* reference events for all normal counters, need assert DEBUG19[21] bit */
 143         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ddrc1_rmw_for_ecc, 12),
 144         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_rreorder, 13),
 145         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_wreorder, 14),
 146         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_0, 15),
 147         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_1, 16),
 148         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_2, 17),
 149         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_3, 18),
 150         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_4, 19),
 151         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_5, 22),
 152         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_6, 23),
 153         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_7, 24),
 154         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_8, 25),
 155         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_9, 26),
 156         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_10, 27),
 157         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_11, 28),
 158         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_12, 31),
 159         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_13, 59),
 160         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_15, 61),
 161         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_29, 63),
 162
 163         /* counter1 specific events */
 164         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_0, 64),
 165         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_1, 65),
 166         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_2, 66),
 167         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_3, 67),
 168         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_4, 68),
 169         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_5, 69),
 170         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_6, 70),
 171         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_7, 71),
 172
 173         /* counter2 specific events */
 174         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_0, 64),
 175         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_1, 65),
 176         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_2, 66),
 177         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_3, 67),
 178         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_4, 68),
 179         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_5, 69),
 180         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_6, 70),
 181         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_7, 71),
 182         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_empty, 72),
 183         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_trans_filt, 73),
 184
 185         /* counter3 specific events */
 186         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_0, 64),
 187         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_1, 65),
 188         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_2, 66),
 189         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_3, 67),
 190         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_4, 68),
 191         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_5, 69),
 192         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_6, 70),
 193         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_7, 71),
 194         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_full, 72),
 195         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pm_wr_trans_filt, 73),
 196
 197         /* counter4 specific events */
 198         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_0, 64),
 199         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_1, 65),
 200         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_2, 66),
 201         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_3, 67),
 202         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_4, 68),
 203         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_5, 69),
 204         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_6, 70),
 205         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_7, 71),
 206         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq2_rmw, 72),
 207         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt, 73),
 208
 209         /* counter5 specific events */
 210         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_0, 64),
 211         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_1, 65),
 212         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_2, 66),
 213         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_3, 67),
 214         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_4, 68),
 215         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_5, 69),
 216         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_6, 70),
 217         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_7, 71),
 218         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq1, 72),
 219
 220         /* counter6 specific events */
 221         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_end_0, 64),
 222         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq2, 72),
 223
 224         /* counter7 specific events */
 225         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_1_2_full, 64),
 226         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_wrq0, 65),
 227
 228         /* counter8 specific events */
 229         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_bias_switched, 64),
 230         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_1_4_full, 65),
 231
 232         /* counter9 specific events */
 233         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_wrq1, 65),
 234         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_3_4_full, 66),
 235
 236         /* counter10 specific events */
 237         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_misc_mrk, 65),
 238         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq0, 66),
 239         NULL,
 240 };
 241
 242 static const struct attribute_group ddr_perf_events_attr_group = {
 243         .name = "events",
 244         .attrs = ddr_perf_events_attrs,
 245 };
 246
 247 PMU_FORMAT_ATTR(event, "config:0-7");
 248 PMU_FORMAT_ATTR(counter, "config:8-15");
 249 PMU_FORMAT_ATTR(axi_id, "config1:0-17");
 250 PMU_FORMAT_ATTR(axi_mask, "config2:0-17");
 251
 252 static struct attribute *ddr_perf_format_attrs[] = {
 253         &format_attr_event.attr,
 254         &format_attr_counter.attr,
 255         &format_attr_axi_id.attr,
 256         &format_attr_axi_mask.attr,
 257         NULL,
 258 };
 259
 260 static const struct attribute_group ddr_perf_format_attr_group = {
 261         .name = "format",
 262         .attrs = ddr_perf_format_attrs,
 263 };
 264
 265 static const struct attribute_group *attr_groups[] = {
 266         &ddr_perf_identifier_attr_group,
 267         &ddr_perf_cpumask_attr_group,
 268         &ddr_perf_events_attr_group,
 269         &ddr_perf_format_attr_group,
 270         NULL,
 271 };
 272
 273 static void ddr_perf_clear_counter(struct ddr_pmu *pmu, int counter)
 274 {
 275         if (counter == CYCLES_COUNTER) {
 276                 writel(0, pmu->base + PMC(counter) + 0x4);
 277                 writel(0, pmu->base + PMC(counter));
 278         } else {
 279                 writel(0, pmu->base + PMC(counter));
 280         }
 281 }
 282
 283 static u64 ddr_perf_read_counter(struct ddr_pmu *pmu, int counter)
 284 {
 285         u32 val_lower, val_upper;
 286         u64 val;
 287
 288         if (counter != CYCLES_COUNTER) {
 289                 val = readl_relaxed(pmu->base + PMC(counter));
 290                 goto out;
 291         }
 292
 293         /* special handling for reading 64bit cycle counter */
 294         do {
 295                 val_upper = readl_relaxed(pmu->base + PMC(counter) + 0x4);
 296                 val_lower = readl_relaxed(pmu->base + PMC(counter));
 297         } while (val_upper != readl_relaxed(pmu->base + PMC(counter) + 0x4));
 298
 299         val = val_upper;
 300         val = (val << 32);
 301         val |= val_lower;
 302 out:
 303         return val;
 304 }
 305
 306 static void ddr_perf_counter_global_config(struct ddr_pmu *pmu, bool enable)
 307 {
 308         u32 ctrl;
 309
 310         ctrl = readl_relaxed(pmu->base + PMGC0);
 311
 312         if (enable) {
 313                 /*
 314                  * The performance monitor must be reset before event counting
 315                  * sequences. The performance monitor can be reset by first freezing
 316                  * one or more counters and then clearing the freeze condition to
 317                  * allow the counters to count according to the settings in the
 318                  * performance monitor registers. Counters can be frozen individually
 319                  * by setting PMLCAn[FC] bits, or simultaneously by setting PMGC0[FAC].
 320                  * Simply clearing these freeze bits will then allow the performance
 321                  * monitor to begin counting based on the register settings.
 322                  */
 323                 ctrl |= PMGC0_FAC;
 324                 writel(ctrl, pmu->base + PMGC0);
 325
 326                 /*
 327                  * Freeze all counters disabled, interrupt enabled, and freeze
 328                  * counters on condition enabled.
 329                  */
 330                 ctrl &= ~PMGC0_FAC;
 331                 ctrl |= PMGC0_PMIE | PMGC0_FCECE;
 332                 writel(ctrl, pmu->base + PMGC0);
 333         } else {
 334                 ctrl |= PMGC0_FAC;
 335                 ctrl &= ~(PMGC0_PMIE | PMGC0_FCECE);
 336                 writel(ctrl, pmu->base + PMGC0);
 337         }
 338 }
 339
 340 static void ddr_perf_counter_local_config(struct ddr_pmu *pmu, int config,
 341                                     int counter, bool enable)
 342 {
 343         u32 ctrl_a;
 344
 345         ctrl_a = readl_relaxed(pmu->base + PMLCA(counter));
 346
 347         if (enable) {
 348                 ctrl_a |= PMLCA_FC;
 349                 writel(ctrl_a, pmu->base + PMLCA(counter));
 350
 351                 ddr_perf_clear_counter(pmu, counter);
 352
 353                 /* Freeze counter disabled, condition enabled, and program event.*/
 354                 ctrl_a &= ~PMLCA_FC;
 355                 ctrl_a |= PMLCA_CE;
 356                 ctrl_a &= ~FIELD_PREP(PMLCA_EVENT, 0x7F);
 357                 ctrl_a |= FIELD_PREP(PMLCA_EVENT, (config & 0x000000FF));
 358                 writel(ctrl_a, pmu->base + PMLCA(counter));
 359         } else {
 360                 /* Freeze counter. */
 361                 ctrl_a |= PMLCA_FC;
 362                 writel(ctrl_a, pmu->base + PMLCA(counter));
 363         }
 364 }
 365
 366 static void ddr_perf_monitor_config(struct ddr_pmu *pmu, int cfg, int cfg1, int cfg2)
 367 {
 368         u32 pmcfg1, pmcfg2;
 369         int event, counter;
 370
 371         event = cfg & 0x000000FF;
 372         counter = (cfg & 0x0000FF00) >> 8;
 373
 374         pmcfg1 = readl_relaxed(pmu->base + PMCFG1);
 375
 376         if (counter == 2 && event == 73)
 377                 pmcfg1 |= PMCFG1_RD_TRANS_FILT_EN;
 378         else if (counter == 2 && event != 73)
 379                 pmcfg1 &= ~PMCFG1_RD_TRANS_FILT_EN;
 380
 381         if (counter == 3 && event == 73)
 382                 pmcfg1 |= PMCFG1_WR_TRANS_FILT_EN;
 383         else if (counter == 3 && event != 73)
 384                 pmcfg1 &= ~PMCFG1_WR_TRANS_FILT_EN;
 385
 386         if (counter == 4 && event == 73)
 387                 pmcfg1 |= PMCFG1_RD_BT_FILT_EN;
 388         else if (counter == 4 && event != 73)
 389                 pmcfg1 &= ~PMCFG1_RD_BT_FILT_EN;
 390
 391         pmcfg1 &= ~FIELD_PREP(PMCFG1_ID_MASK, 0x3FFFF);
 392         pmcfg1 |= FIELD_PREP(PMCFG1_ID_MASK, cfg2);
 393         writel(pmcfg1, pmu->base + PMCFG1);
 394
 395         pmcfg2 = readl_relaxed(pmu->base + PMCFG2);
 396         pmcfg2 &= ~FIELD_PREP(PMCFG2_ID, 0x3FFFF);
 397         pmcfg2 |= FIELD_PREP(PMCFG2_ID, cfg1);
 398         writel(pmcfg2, pmu->base + PMCFG2);
 399 }
 400
 401 static void ddr_perf_event_update(struct perf_event *event)
 402 {
 403         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 404         struct hw_perf_event *hwc = &event->hw;
 405         int counter = hwc->idx;
 406         u64 new_raw_count;
 407
 408         new_raw_count = ddr_perf_read_counter(pmu, counter);
 409         local64_add(new_raw_count, &event->count);
 410
 411         /* clear counter's value every time */
 412         ddr_perf_clear_counter(pmu, counter);
 413 }
 414
 415 static int ddr_perf_event_init(struct perf_event *event)
 416 {
 417         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 418         struct hw_perf_event *hwc = &event->hw;
 419         struct perf_event *sibling;
 420
 421         if (event->attr.type != event->pmu->type)
 422                 return -ENOENT;
 423
 424         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 425                 return -EOPNOTSUPP;
 426
 427         if (event->cpu < 0) {
 428                 dev_warn(pmu->dev, "Can't provide per-task data!\n");
 429                 return -EOPNOTSUPP;
 430         }
 431
 432         /*
 433          * We must NOT create groups containing mixed PMUs, although software
 434          * events are acceptable (for example to create a CCN group
 435          * periodically read when a hrtimer aka cpu-clock leader triggers).
 436          */
 437         if (event->group_leader->pmu != event->pmu &&
 438                         !is_software_event(event->group_leader))
 439                 return -EINVAL;
 440
 441         for_each_sibling_event(sibling, event->group_leader) {
 442                 if (sibling->pmu != event->pmu &&
 443                                 !is_software_event(sibling))
 444                         return -EINVAL;
 445         }
 446
 447         event->cpu = pmu->cpu;
 448         hwc->idx = -1;
 449
 450         return 0;
 451 }
 452
 453 static void ddr_perf_event_start(struct perf_event *event, int flags)
 454 {
 455         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 456         struct hw_perf_event *hwc = &event->hw;
 457         int counter = hwc->idx;
 458
 459         local64_set(&hwc->prev_count, 0);
 460
 461         ddr_perf_counter_local_config(pmu, event->attr.config, counter, true);
 462         hwc->state = 0;
 463 }
 464
 465 static int ddr_perf_event_add(struct perf_event *event, int flags)
 466 {
 467         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 468         struct hw_perf_event *hwc = &event->hw;
 469         int cfg = event->attr.config;
 470         int cfg1 = event->attr.config1;
 471         int cfg2 = event->attr.config2;
 472         int counter;
 473
 474         counter = (cfg & 0x0000FF00) >> 8;
 475
 476         pmu->events[counter] = event;
 477         pmu->active_events++;
 478         hwc->idx = counter;
 479         hwc->state |= PERF_HES_STOPPED;
 480
 481         if (flags & PERF_EF_START)
 482                 ddr_perf_event_start(event, flags);
 483
 484         /* read trans, write trans, read beat */
 485         ddr_perf_monitor_config(pmu, cfg, cfg1, cfg2);
 486
 487         return 0;
 488 }
 489
 490 static void ddr_perf_event_stop(struct perf_event *event, int flags)
 491 {
 492         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 493         struct hw_perf_event *hwc = &event->hw;
 494         int counter = hwc->idx;
 495
 496         ddr_perf_counter_local_config(pmu, event->attr.config, counter, false);
 497         ddr_perf_event_update(event);
 498
 499         hwc->state |= PERF_HES_STOPPED;
 500 }
 501
 502 static void ddr_perf_event_del(struct perf_event *event, int flags)
 503 {
 504         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 505         struct hw_perf_event *hwc = &event->hw;
 506
 507         ddr_perf_event_stop(event, PERF_EF_UPDATE);
 508
 509         pmu->active_events--;
 510         hwc->idx = -1;
 511 }
 512
 513 static void ddr_perf_pmu_enable(struct pmu *pmu)
 514 {
 515         struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
 516
 517         ddr_perf_counter_global_config(ddr_pmu, true);
 518 }
 519
 520 static void ddr_perf_pmu_disable(struct pmu *pmu)
 521 {
 522         struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
 523
 524         ddr_perf_counter_global_config(ddr_pmu, false);
 525 }
 526
 527 static void ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base,
 528                          struct device *dev)
 529 {
 530         *pmu = (struct ddr_pmu) {
 531                 .pmu = (struct pmu) {
 532                         .module       = THIS_MODULE,
 533                         .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
 534                         .task_ctx_nr  = perf_invalid_context,
 535                         .attr_groups  = attr_groups,
 536                         .event_init   = ddr_perf_event_init,
 537                         .add          = ddr_perf_event_add,
 538                         .del          = ddr_perf_event_del,
 539                         .start        = ddr_perf_event_start,
 540                         .stop         = ddr_perf_event_stop,
 541                         .read         = ddr_perf_event_update,
 542                         .pmu_enable   = ddr_perf_pmu_enable,
 543                         .pmu_disable  = ddr_perf_pmu_disable,
 544                 },
 545                 .base = base,
 546                 .dev = dev,
 547         };
 548 }
 549
 550 static irqreturn_t ddr_perf_irq_handler(int irq, void *p)
 551 {
 552         struct ddr_pmu *pmu = (struct ddr_pmu *)p;
 553         struct perf_event *event;
 554         int i;
 555
 556         /*
 557          * Counters can generate an interrupt on an overflow when msb of a
 558          * counter changes from 0 to 1. For the interrupt to be signalled,
 559          * below condition mush be satisfied:
 560          * PMGC0[PMIE] = 1, PMGC0[FCECE] = 1, PMLCAn[CE] = 1
 561          * When an interrupt is signalled, PMGC0[FAC] is set by hardware and
 562          * all of the registers are frozen.
 563          * Software can clear the interrupt condition by resetting the performance
 564          * monitor and clearing the most significant bit of the counter that
 565          * generate the overflow.
 566          */
 567         for (i = 0; i < NUM_COUNTERS; i++) {
 568                 if (!pmu->events[i])
 569                         continue;
 570
 571                 event = pmu->events[i];
 572
 573                 ddr_perf_event_update(event);
 574         }
 575
 576         ddr_perf_counter_global_config(pmu, true);
 577
 578         return IRQ_HANDLED;
 579 }
 580
 581 static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
 582 {
 583         struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);
 584         int target;
 585
 586         if (cpu != pmu->cpu)
 587                 return 0;
 588
 589         target = cpumask_any_but(cpu_online_mask, cpu);
 590         if (target >= nr_cpu_ids)
 591                 return 0;
 592
 593         perf_pmu_migrate_context(&pmu->pmu, cpu, target);
 594         pmu->cpu = target;
 595
 596         WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)));
 597
 598         return 0;
 599 }
 600
 601 static int ddr_perf_probe(struct platform_device *pdev)
 602 {
 603         struct ddr_pmu *pmu;
 604         void __iomem *base;
 605         int ret, irq;
 606         char *name;
 607
 608         base = devm_platform_ioremap_resource(pdev, 0);
 609         if (IS_ERR(base))
 610                 return PTR_ERR(base);
 611
 612         pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);
 613         if (!pmu)
 614                 return -ENOMEM;
 615
 616         ddr_perf_init(pmu, base, &pdev->dev);
 617
 618         pmu->devtype_data = of_device_get_match_data(&pdev->dev);
 619
 620         platform_set_drvdata(pdev, pmu);
 621
 622         pmu->id = ida_simple_get(&ddr_ida, 0, 0, GFP_KERNEL);
 623         name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", pmu->id);
 624         if (!name) {
 625                 ret = -ENOMEM;
 626                 goto format_string_err;
 627         }
 628
 629         pmu->cpu = raw_smp_processor_id();
 630         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DDR_CPUHP_CB_NAME,
 631                                       NULL, ddr_perf_offline_cpu);
 632         if (ret < 0) {
 633                 dev_err(&pdev->dev, "Failed to add callbacks for multi state\n");
 634                 goto cpuhp_state_err;
 635         }
 636         pmu->cpuhp_state = ret;
 637
 638         /* Register the pmu instance for cpu hotplug */
 639         ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 640         if (ret) {
 641                 dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
 642                 goto cpuhp_instance_err;
 643         }
 644
 645         /* Request irq */
 646         irq = platform_get_irq(pdev, 0);
 647         if (irq < 0) {
 648                 ret = irq;
 649                 goto ddr_perf_err;
 650         }
 651
 652         ret = devm_request_irq(&pdev->dev, irq, ddr_perf_irq_handler,
 653                                IRQF_NOBALANCING | IRQF_NO_THREAD,
 654                                DDR_CPUHP_CB_NAME, pmu);
 655         if (ret < 0) {
 656                 dev_err(&pdev->dev, "Request irq failed: %d", ret);
 657                 goto ddr_perf_err;
 658         }
 659
 660         pmu->irq = irq;
 661         ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu));
 662         if (ret) {
 663                 dev_err(pmu->dev, "Failed to set interrupt affinity\n");
 664                 goto ddr_perf_err;
 665         }
 666
 667         ret = perf_pmu_register(&pmu->pmu, name, -1);
 668         if (ret)
 669                 goto ddr_perf_err;
 670
 671         return 0;
 672
 673 ddr_perf_err:
 674         cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 675 cpuhp_instance_err:
 676         cpuhp_remove_multi_state(pmu->cpuhp_state);
 677 cpuhp_state_err:
 678 format_string_err:
 679         ida_simple_remove(&ddr_ida, pmu->id);
 680         dev_warn(&pdev->dev, "i.MX9 DDR Perf PMU failed (%d), disabled\n", ret);
 681         return ret;
 682 }
 683
 684 static int ddr_perf_remove(struct platform_device *pdev)
 685 {
 686         struct ddr_pmu *pmu = platform_get_drvdata(pdev);
 687
 688         cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 689         cpuhp_remove_multi_state(pmu->cpuhp_state);
 690
 691         perf_pmu_unregister(&pmu->pmu);
 692
 693         ida_simple_remove(&ddr_ida, pmu->id);
 694
 695         return 0;
 696 }
 697
 698 static struct platform_driver imx_ddr_pmu_driver = {
 699         .driver         = {
 700                 .name                = "imx9-ddr-pmu",
 701                 .of_match_table      = imx_ddr_pmu_dt_ids,
 702                 .suppress_bind_attrs = true,
 703         },
 704         .probe          = ddr_perf_probe,
 705         .remove         = ddr_perf_remove,
 706 };
 707 module_platform_driver(imx_ddr_pmu_driver);
 708
 709 MODULE_AUTHOR("Xu Yang <[email protected]>");
 710 MODULE_LICENSE("GPL v2");
 711 MODULE_DESCRIPTION("DDRC PerfMon for i.MX9 SoCs");