drivers/perf/fsl_imx9_ddr_perf.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright 2023 NXP
   3
   4 #include <linux/bitfield.h>
   5 #include <linux/init.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/io.h>
   8 #include <linux/module.h>
   9 #include <linux/of.h>
  10 #include <linux/platform_device.h>
  11 #include <linux/perf_event.h>
  12
  13 /* Performance monitor configuration */
  14 #define PMCFG1                          0x00
  15 #define MX93_PMCFG1_RD_TRANS_FILT_EN    BIT(31)
  16 #define MX93_PMCFG1_WR_TRANS_FILT_EN    BIT(30)
  17 #define MX93_PMCFG1_RD_BT_FILT_EN       BIT(29)
  18 #define MX93_PMCFG1_ID_MASK             GENMASK(17, 0)
  19
  20 #define MX95_PMCFG1_WR_BEAT_FILT_EN     BIT(31)
  21 #define MX95_PMCFG1_RD_BEAT_FILT_EN     BIT(30)
  22
  23 #define PMCFG2                          0x04
  24 #define MX93_PMCFG2_ID                  GENMASK(17, 0)
  25
  26 #define PMCFG3                          0x08
  27 #define PMCFG4                          0x0C
  28 #define PMCFG5                          0x10
  29 #define PMCFG6                          0x14
  30 #define MX95_PMCFG_ID_MASK              GENMASK(9, 0)
  31 #define MX95_PMCFG_ID                   GENMASK(25, 16)
  32
  33 /* Global control register affects all counters and takes priority over local control registers */
  34 #define PMGC0           0x40
  35 /* Global control register bits */
  36 #define PMGC0_FAC       BIT(31)
  37 #define PMGC0_PMIE      BIT(30)
  38 #define PMGC0_FCECE     BIT(29)
  39
  40 /*
  41  * 64bit counter0 exclusively dedicated to counting cycles
  42  * 32bit counters monitor counter-specific events in addition to counting reference events
  43  */
  44 #define PMLCA(n)        (0x40 + 0x10 + (0x10 * n))
  45 #define PMLCB(n)        (0x40 + 0x14 + (0x10 * n))
  46 #define PMC(n)          (0x40 + 0x18 + (0x10 * n))
  47 /* Local control register bits */
  48 #define PMLCA_FC        BIT(31)
  49 #define PMLCA_CE        BIT(26)
  50 #define PMLCA_EVENT     GENMASK(22, 16)
  51
  52 #define NUM_COUNTERS            11
  53 #define CYCLES_COUNTER          0
  54 #define CYCLES_EVENT_ID         0
  55
  56 #define CONFIG_EVENT_MASK       GENMASK(7, 0)
  57 #define CONFIG_COUNTER_MASK     GENMASK(23, 16)
  58
  59 #define to_ddr_pmu(p)           container_of(p, struct ddr_pmu, pmu)
  60
  61 #define DDR_PERF_DEV_NAME       "imx9_ddr"
  62 #define DDR_CPUHP_CB_NAME       DDR_PERF_DEV_NAME "_perf_pmu"
  63
  64 static DEFINE_IDA(ddr_ida);
  65
  66 struct imx_ddr_devtype_data {
  67         const char *identifier;         /* system PMU identifier for userspace */
  68 };
  69
  70 struct ddr_pmu {
  71         struct pmu pmu;
  72         void __iomem *base;
  73         unsigned int cpu;
  74         struct hlist_node node;
  75         struct device *dev;
  76         struct perf_event *events[NUM_COUNTERS];
  77         int active_events;
  78         enum cpuhp_state cpuhp_state;
  79         const struct imx_ddr_devtype_data *devtype_data;
  80         int irq;
  81         int id;
  82 };
  83
  84 static const struct imx_ddr_devtype_data imx93_devtype_data = {
  85         .identifier = "imx93",
  86 };
  87
  88 static const struct imx_ddr_devtype_data imx95_devtype_data = {
  89         .identifier = "imx95",
  90 };
  91
  92 static inline bool is_imx93(struct ddr_pmu *pmu)
  93 {
  94         return pmu->devtype_data == &imx93_devtype_data;
  95 }
  96
  97 static inline bool is_imx95(struct ddr_pmu *pmu)
  98 {
  99         return pmu->devtype_data == &imx95_devtype_data;
 100 }
 101
 102 static const struct of_device_id imx_ddr_pmu_dt_ids[] = {
 103         { .compatible = "fsl,imx93-ddr-pmu", .data = &imx93_devtype_data },
 104         { .compatible = "fsl,imx95-ddr-pmu", .data = &imx95_devtype_data },
 105         { /* sentinel */ }
 106 };
 107 MODULE_DEVICE_TABLE(of, imx_ddr_pmu_dt_ids);
 108
 109 static ssize_t ddr_perf_identifier_show(struct device *dev,
 110                                         struct device_attribute *attr,
 111                                         char *page)
 112 {
 113         struct ddr_pmu *pmu = dev_get_drvdata(dev);
 114
 115         return sysfs_emit(page, "%s\n", pmu->devtype_data->identifier);
 116 }
 117
 118 static struct device_attribute ddr_perf_identifier_attr =
 119         __ATTR(identifier, 0444, ddr_perf_identifier_show, NULL);
 120
 121 static struct attribute *ddr_perf_identifier_attrs[] = {
 122         &ddr_perf_identifier_attr.attr,
 123         NULL,
 124 };
 125
 126 static struct attribute_group ddr_perf_identifier_attr_group = {
 127         .attrs = ddr_perf_identifier_attrs,
 128 };
 129
 130 static ssize_t ddr_perf_cpumask_show(struct device *dev,
 131                                      struct device_attribute *attr, char *buf)
 132 {
 133         struct ddr_pmu *pmu = dev_get_drvdata(dev);
 134
 135         return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
 136 }
 137
 138 static struct device_attribute ddr_perf_cpumask_attr =
 139         __ATTR(cpumask, 0444, ddr_perf_cpumask_show, NULL);
 140
 141 static struct attribute *ddr_perf_cpumask_attrs[] = {
 142         &ddr_perf_cpumask_attr.attr,
 143         NULL,
 144 };
 145
 146 static const struct attribute_group ddr_perf_cpumask_attr_group = {
 147         .attrs = ddr_perf_cpumask_attrs,
 148 };
 149
 150 struct imx9_pmu_events_attr {
 151         struct device_attribute attr;
 152         u64 id;
 153         const void *devtype_data;
 154 };
 155
 156 static ssize_t ddr_pmu_event_show(struct device *dev,
 157                                   struct device_attribute *attr, char *page)
 158 {
 159         struct imx9_pmu_events_attr *pmu_attr;
 160
 161         pmu_attr = container_of(attr, struct imx9_pmu_events_attr, attr);
 162         return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
 163 }
 164
 165 #define COUNTER_OFFSET_IN_EVENT 8
 166 #define ID(counter, id) ((counter << COUNTER_OFFSET_IN_EVENT) | id)
 167
 168 #define DDR_PMU_EVENT_ATTR_COMM(_name, _id, _data)                      \
 169         (&((struct imx9_pmu_events_attr[]) {                            \
 170                 { .attr = __ATTR(_name, 0444, ddr_pmu_event_show, NULL),\
 171                   .id = _id,                                            \
 172                   .devtype_data = _data, }                              \
 173         })[0].attr.attr)
 174
 175 #define IMX9_DDR_PMU_EVENT_ATTR(_name, _id)                             \
 176         DDR_PMU_EVENT_ATTR_COMM(_name, _id, NULL)
 177
 178 #define IMX93_DDR_PMU_EVENT_ATTR(_name, _id)                            \
 179         DDR_PMU_EVENT_ATTR_COMM(_name, _id, &imx93_devtype_data)
 180
 181 #define IMX95_DDR_PMU_EVENT_ATTR(_name, _id)                            \
 182         DDR_PMU_EVENT_ATTR_COMM(_name, _id, &imx95_devtype_data)
 183
 184 static struct attribute *ddr_perf_events_attrs[] = {
 185         /* counter0 cycles event */
 186         IMX9_DDR_PMU_EVENT_ATTR(cycles, 0),
 187
 188         /* reference events for all normal counters, need assert DEBUG19[21] bit */
 189         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ddrc1_rmw_for_ecc, 12),
 190         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_rreorder, 13),
 191         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_wreorder, 14),
 192         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_0, 15),
 193         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_1, 16),
 194         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_2, 17),
 195         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_3, 18),
 196         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_4, 19),
 197         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_5, 22),
 198         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_6, 23),
 199         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_7, 24),
 200         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_8, 25),
 201         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_9, 26),
 202         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_10, 27),
 203         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_11, 28),
 204         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_12, 31),
 205         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_13, 59),
 206         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_15, 61),
 207         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_29, 63),
 208
 209         /* counter1 specific events */
 210         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_0, ID(1, 64)),
 211         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_1, ID(1, 65)),
 212         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_2, ID(1, 66)),
 213         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_3, ID(1, 67)),
 214         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_4, ID(1, 68)),
 215         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_5, ID(1, 69)),
 216         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_6, ID(1, 70)),
 217         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_7, ID(1, 71)),
 218
 219         /* counter2 specific events */
 220         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_0, ID(2, 64)),
 221         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_1, ID(2, 65)),
 222         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_2, ID(2, 66)),
 223         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_3, ID(2, 67)),
 224         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_4, ID(2, 68)),
 225         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_5, ID(2, 69)),
 226         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_6, ID(2, 70)),
 227         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_7, ID(2, 71)),
 228         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_empty, ID(2, 72)),
 229         IMX93_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_trans_filt, ID(2, 73)),   /* imx93 specific*/
 230         IMX95_DDR_PMU_EVENT_ATTR(eddrtq_pm_wr_beat_filt, ID(2, 73)),    /* imx95 specific*/
 231
 232         /* counter3 specific events */
 233         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_0, ID(3, 64)),
 234         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_1, ID(3, 65)),
 235         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_2, ID(3, 66)),
 236         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_3, ID(3, 67)),
 237         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_4, ID(3, 68)),
 238         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_5, ID(3, 69)),
 239         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_6, ID(3, 70)),
 240         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_7, ID(3, 71)),
 241         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_full, ID(3, 72)),
 242         IMX93_DDR_PMU_EVENT_ATTR(eddrtq_pm_wr_trans_filt, ID(3, 73)),   /* imx93 specific*/
 243         IMX95_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt2, ID(3, 73)),   /* imx95 specific*/
 244
 245         /* counter4 specific events */
 246         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_0, ID(4, 64)),
 247         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_1, ID(4, 65)),
 248         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_2, ID(4, 66)),
 249         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_3, ID(4, 67)),
 250         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_4, ID(4, 68)),
 251         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_5, ID(4, 69)),
 252         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_6, ID(4, 70)),
 253         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_7, ID(4, 71)),
 254         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq2_rmw, ID(4, 72)),
 255         IMX93_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt, ID(4, 73)),    /* imx93 specific*/
 256         IMX95_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt1, ID(4, 73)),   /* imx95 specific*/
 257
 258         /* counter5 specific events */
 259         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_0, ID(5, 64)),
 260         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_1, ID(5, 65)),
 261         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_2, ID(5, 66)),
 262         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_3, ID(5, 67)),
 263         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_4, ID(5, 68)),
 264         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_5, ID(5, 69)),
 265         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_6, ID(5, 70)),
 266         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_7, ID(5, 71)),
 267         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq1, ID(5, 72)),
 268         IMX95_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt0, ID(5, 73)),   /* imx95 specific*/
 269
 270         /* counter6 specific events */
 271         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_end_0, ID(6, 64)),
 272         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq2, ID(6, 72)),
 273
 274         /* counter7 specific events */
 275         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_1_2_full, ID(7, 64)),
 276         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_wrq0, ID(7, 65)),
 277
 278         /* counter8 specific events */
 279         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_bias_switched, ID(8, 64)),
 280         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_1_4_full, ID(8, 65)),
 281
 282         /* counter9 specific events */
 283         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_wrq1, ID(9, 65)),
 284         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_3_4_full, ID(9, 66)),
 285
 286         /* counter10 specific events */
 287         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_misc_mrk, ID(10, 65)),
 288         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq0, ID(10, 66)),
 289         NULL,
 290 };
 291
 292 static umode_t
 293 ddr_perf_events_attrs_is_visible(struct kobject *kobj,
 294                                        struct attribute *attr, int unused)
 295 {
 296         struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
 297         struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
 298         struct imx9_pmu_events_attr *eattr;
 299
 300         eattr = container_of(attr, typeof(*eattr), attr.attr);
 301
 302         if (!eattr->devtype_data)
 303                 return attr->mode;
 304
 305         if (eattr->devtype_data != ddr_pmu->devtype_data)
 306                 return 0;
 307
 308         return attr->mode;
 309 }
 310
 311 static const struct attribute_group ddr_perf_events_attr_group = {
 312         .name = "events",
 313         .attrs = ddr_perf_events_attrs,
 314         .is_visible = ddr_perf_events_attrs_is_visible,
 315 };
 316
 317 PMU_FORMAT_ATTR(event, "config:0-7,16-23");
 318 PMU_FORMAT_ATTR(counter, "config:8-15");
 319 PMU_FORMAT_ATTR(axi_id, "config1:0-17");
 320 PMU_FORMAT_ATTR(axi_mask, "config2:0-17");
 321
 322 static struct attribute *ddr_perf_format_attrs[] = {
 323         &format_attr_event.attr,
 324         &format_attr_counter.attr,
 325         &format_attr_axi_id.attr,
 326         &format_attr_axi_mask.attr,
 327         NULL,
 328 };
 329
 330 static const struct attribute_group ddr_perf_format_attr_group = {
 331         .name = "format",
 332         .attrs = ddr_perf_format_attrs,
 333 };
 334
 335 static const struct attribute_group *attr_groups[] = {
 336         &ddr_perf_identifier_attr_group,
 337         &ddr_perf_cpumask_attr_group,
 338         &ddr_perf_events_attr_group,
 339         &ddr_perf_format_attr_group,
 340         NULL,
 341 };
 342
 343 static void ddr_perf_clear_counter(struct ddr_pmu *pmu, int counter)
 344 {
 345         if (counter == CYCLES_COUNTER) {
 346                 writel(0, pmu->base + PMC(counter) + 0x4);
 347                 writel(0, pmu->base + PMC(counter));
 348         } else {
 349                 writel(0, pmu->base + PMC(counter));
 350         }
 351 }
 352
 353 static u64 ddr_perf_read_counter(struct ddr_pmu *pmu, int counter)
 354 {
 355         u32 val_lower, val_upper;
 356         u64 val;
 357
 358         if (counter != CYCLES_COUNTER) {
 359                 val = readl_relaxed(pmu->base + PMC(counter));
 360                 goto out;
 361         }
 362
 363         /* special handling for reading 64bit cycle counter */
 364         do {
 365                 val_upper = readl_relaxed(pmu->base + PMC(counter) + 0x4);
 366                 val_lower = readl_relaxed(pmu->base + PMC(counter));
 367         } while (val_upper != readl_relaxed(pmu->base + PMC(counter) + 0x4));
 368
 369         val = val_upper;
 370         val = (val << 32);
 371         val |= val_lower;
 372 out:
 373         return val;
 374 }
 375
 376 static void ddr_perf_counter_global_config(struct ddr_pmu *pmu, bool enable)
 377 {
 378         u32 ctrl;
 379
 380         ctrl = readl_relaxed(pmu->base + PMGC0);
 381
 382         if (enable) {
 383                 /*
 384                  * The performance monitor must be reset before event counting
 385                  * sequences. The performance monitor can be reset by first freezing
 386                  * one or more counters and then clearing the freeze condition to
 387                  * allow the counters to count according to the settings in the
 388                  * performance monitor registers. Counters can be frozen individually
 389                  * by setting PMLCAn[FC] bits, or simultaneously by setting PMGC0[FAC].
 390                  * Simply clearing these freeze bits will then allow the performance
 391                  * monitor to begin counting based on the register settings.
 392                  */
 393                 ctrl |= PMGC0_FAC;
 394                 writel(ctrl, pmu->base + PMGC0);
 395
 396                 /*
 397                  * Freeze all counters disabled, interrupt enabled, and freeze
 398                  * counters on condition enabled.
 399                  */
 400                 ctrl &= ~PMGC0_FAC;
 401                 ctrl |= PMGC0_PMIE | PMGC0_FCECE;
 402                 writel(ctrl, pmu->base + PMGC0);
 403         } else {
 404                 ctrl |= PMGC0_FAC;
 405                 ctrl &= ~(PMGC0_PMIE | PMGC0_FCECE);
 406                 writel(ctrl, pmu->base + PMGC0);
 407         }
 408 }
 409
 410 static void ddr_perf_counter_local_config(struct ddr_pmu *pmu, int config,
 411                                     int counter, bool enable)
 412 {
 413         u32 ctrl_a;
 414         int event;
 415
 416         ctrl_a = readl_relaxed(pmu->base + PMLCA(counter));
 417         event = FIELD_GET(CONFIG_EVENT_MASK, config);
 418
 419         if (enable) {
 420                 ctrl_a |= PMLCA_FC;
 421                 writel(ctrl_a, pmu->base + PMLCA(counter));
 422
 423                 ddr_perf_clear_counter(pmu, counter);
 424
 425                 /* Freeze counter disabled, condition enabled, and program event.*/
 426                 ctrl_a &= ~PMLCA_FC;
 427                 ctrl_a |= PMLCA_CE;
 428                 ctrl_a &= ~FIELD_PREP(PMLCA_EVENT, 0x7F);
 429                 ctrl_a |= FIELD_PREP(PMLCA_EVENT, event);
 430                 writel(ctrl_a, pmu->base + PMLCA(counter));
 431         } else {
 432                 /* Freeze counter. */
 433                 ctrl_a |= PMLCA_FC;
 434                 writel(ctrl_a, pmu->base + PMLCA(counter));
 435         }
 436 }
 437
 438 static void imx93_ddr_perf_monitor_config(struct ddr_pmu *pmu, int event,
 439                                           int counter, int axi_id, int axi_mask)
 440 {
 441         u32 pmcfg1, pmcfg2;
 442         u32 mask[] = {  MX93_PMCFG1_RD_TRANS_FILT_EN,
 443                         MX93_PMCFG1_WR_TRANS_FILT_EN,
 444                         MX93_PMCFG1_RD_BT_FILT_EN };
 445
 446         pmcfg1 = readl_relaxed(pmu->base + PMCFG1);
 447
 448         if (counter >= 2 && counter <= 4)
 449                 pmcfg1 = event == 73 ? pmcfg1 | mask[counter - 2] :
 450                                 pmcfg1 & ~mask[counter - 2];
 451
 452         pmcfg1 &= ~FIELD_PREP(MX93_PMCFG1_ID_MASK, 0x3FFFF);
 453         pmcfg1 |= FIELD_PREP(MX93_PMCFG1_ID_MASK, axi_mask);
 454         writel_relaxed(pmcfg1, pmu->base + PMCFG1);
 455
 456         pmcfg2 = readl_relaxed(pmu->base + PMCFG2);
 457         pmcfg2 &= ~FIELD_PREP(MX93_PMCFG2_ID, 0x3FFFF);
 458         pmcfg2 |= FIELD_PREP(MX93_PMCFG2_ID, axi_id);
 459         writel_relaxed(pmcfg2, pmu->base + PMCFG2);
 460 }
 461
 462 static void imx95_ddr_perf_monitor_config(struct ddr_pmu *pmu, int event,
 463                                           int counter, int axi_id, int axi_mask)
 464 {
 465         u32 pmcfg1, pmcfg, offset = 0;
 466
 467         pmcfg1 = readl_relaxed(pmu->base + PMCFG1);
 468
 469         if (event == 73) {
 470                 switch (counter) {
 471                 case 2:
 472                         pmcfg1 |= MX95_PMCFG1_WR_BEAT_FILT_EN;
 473                         offset = PMCFG3;
 474                         break;
 475                 case 3:
 476                         pmcfg1 |= MX95_PMCFG1_RD_BEAT_FILT_EN;
 477                         offset = PMCFG4;
 478                         break;
 479                 case 4:
 480                         pmcfg1 |= MX95_PMCFG1_RD_BEAT_FILT_EN;
 481                         offset = PMCFG5;
 482                         break;
 483                 case 5:
 484                         pmcfg1 |= MX95_PMCFG1_RD_BEAT_FILT_EN;
 485                         offset = PMCFG6;
 486                         break;
 487                 }
 488         } else {
 489                 switch (counter) {
 490                 case 2:
 491                         pmcfg1 &= ~MX95_PMCFG1_WR_BEAT_FILT_EN;
 492                         break;
 493                 case 3:
 494                 case 4:
 495                 case 5:
 496                         pmcfg1 &= ~MX95_PMCFG1_RD_BEAT_FILT_EN;
 497                         break;
 498                 }
 499         }
 500
 501         writel_relaxed(pmcfg1, pmu->base + PMCFG1);
 502
 503         if (offset) {
 504                 pmcfg = readl_relaxed(pmu->base + offset);
 505                 pmcfg &= ~(FIELD_PREP(MX95_PMCFG_ID_MASK, 0x3FF) |
 506                            FIELD_PREP(MX95_PMCFG_ID, 0x3FF));
 507                 pmcfg |= (FIELD_PREP(MX95_PMCFG_ID_MASK, axi_mask) |
 508                           FIELD_PREP(MX95_PMCFG_ID, axi_id));
 509                 writel_relaxed(pmcfg, pmu->base + offset);
 510         }
 511 }
 512
 513 static void ddr_perf_event_update(struct perf_event *event)
 514 {
 515         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 516         struct hw_perf_event *hwc = &event->hw;
 517         int counter = hwc->idx;
 518         u64 new_raw_count;
 519
 520         new_raw_count = ddr_perf_read_counter(pmu, counter);
 521         local64_add(new_raw_count, &event->count);
 522
 523         /* clear counter's value every time */
 524         ddr_perf_clear_counter(pmu, counter);
 525 }
 526
 527 static int ddr_perf_event_init(struct perf_event *event)
 528 {
 529         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 530         struct hw_perf_event *hwc = &event->hw;
 531         struct perf_event *sibling;
 532
 533         if (event->attr.type != event->pmu->type)
 534                 return -ENOENT;
 535
 536         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 537                 return -EOPNOTSUPP;
 538
 539         if (event->cpu < 0) {
 540                 dev_warn(pmu->dev, "Can't provide per-task data!\n");
 541                 return -EOPNOTSUPP;
 542         }
 543
 544         /*
 545          * We must NOT create groups containing mixed PMUs, although software
 546          * events are acceptable (for example to create a CCN group
 547          * periodically read when a hrtimer aka cpu-clock leader triggers).
 548          */
 549         if (event->group_leader->pmu != event->pmu &&
 550                         !is_software_event(event->group_leader))
 551                 return -EINVAL;
 552
 553         for_each_sibling_event(sibling, event->group_leader) {
 554                 if (sibling->pmu != event->pmu &&
 555                                 !is_software_event(sibling))
 556                         return -EINVAL;
 557         }
 558
 559         event->cpu = pmu->cpu;
 560         hwc->idx = -1;
 561
 562         return 0;
 563 }
 564
 565 static void ddr_perf_event_start(struct perf_event *event, int flags)
 566 {
 567         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 568         struct hw_perf_event *hwc = &event->hw;
 569         int counter = hwc->idx;
 570
 571         local64_set(&hwc->prev_count, 0);
 572
 573         ddr_perf_counter_local_config(pmu, event->attr.config, counter, true);
 574         hwc->state = 0;
 575 }
 576
 577 static int ddr_perf_alloc_counter(struct ddr_pmu *pmu, int event, int counter)
 578 {
 579         int i;
 580
 581         if (event == CYCLES_EVENT_ID) {
 582                 // Cycles counter is dedicated for cycle event.
 583                 if (pmu->events[CYCLES_COUNTER] == NULL)
 584                         return CYCLES_COUNTER;
 585         } else if (counter != 0) {
 586                 // Counter specific event use specific counter.
 587                 if (pmu->events[counter] == NULL)
 588                         return counter;
 589         } else {
 590                 // Auto allocate counter for referene event.
 591                 for (i = 1; i < NUM_COUNTERS; i++)
 592                         if (pmu->events[i] == NULL)
 593                                 return i;
 594         }
 595
 596         return -ENOENT;
 597 }
 598
 599 static int ddr_perf_event_add(struct perf_event *event, int flags)
 600 {
 601         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 602         struct hw_perf_event *hwc = &event->hw;
 603         int cfg = event->attr.config;
 604         int cfg1 = event->attr.config1;
 605         int cfg2 = event->attr.config2;
 606         int event_id, counter;
 607
 608         event_id = FIELD_GET(CONFIG_EVENT_MASK, cfg);
 609         counter = FIELD_GET(CONFIG_COUNTER_MASK, cfg);
 610
 611         counter = ddr_perf_alloc_counter(pmu, event_id, counter);
 612         if (counter < 0) {
 613                 dev_dbg(pmu->dev, "There are not enough counters\n");
 614                 return -EOPNOTSUPP;
 615         }
 616
 617         pmu->events[counter] = event;
 618         pmu->active_events++;
 619         hwc->idx = counter;
 620         hwc->state |= PERF_HES_STOPPED;
 621
 622         if (is_imx93(pmu))
 623                 /* read trans, write trans, read beat */
 624                 imx93_ddr_perf_monitor_config(pmu, event_id, counter, cfg1, cfg2);
 625
 626         if (is_imx95(pmu))
 627                 /* write beat, read beat2, read beat1, read beat */
 628                 imx95_ddr_perf_monitor_config(pmu, event_id, counter, cfg1, cfg2);
 629
 630         if (flags & PERF_EF_START)
 631                 ddr_perf_event_start(event, flags);
 632
 633         return 0;
 634 }
 635
 636 static void ddr_perf_event_stop(struct perf_event *event, int flags)
 637 {
 638         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 639         struct hw_perf_event *hwc = &event->hw;
 640         int counter = hwc->idx;
 641
 642         ddr_perf_counter_local_config(pmu, event->attr.config, counter, false);
 643         ddr_perf_event_update(event);
 644
 645         hwc->state |= PERF_HES_STOPPED;
 646 }
 647
 648 static void ddr_perf_event_del(struct perf_event *event, int flags)
 649 {
 650         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 651         struct hw_perf_event *hwc = &event->hw;
 652         int counter = hwc->idx;
 653
 654         ddr_perf_event_stop(event, PERF_EF_UPDATE);
 655
 656         pmu->events[counter] = NULL;
 657         pmu->active_events--;
 658         hwc->idx = -1;
 659 }
 660
 661 static void ddr_perf_pmu_enable(struct pmu *pmu)
 662 {
 663         struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
 664
 665         ddr_perf_counter_global_config(ddr_pmu, true);
 666 }
 667
 668 static void ddr_perf_pmu_disable(struct pmu *pmu)
 669 {
 670         struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
 671
 672         ddr_perf_counter_global_config(ddr_pmu, false);
 673 }
 674
 675 static void ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base,
 676                          struct device *dev)
 677 {
 678         *pmu = (struct ddr_pmu) {
 679                 .pmu = (struct pmu) {
 680                         .module       = THIS_MODULE,
 681                         .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
 682                         .task_ctx_nr  = perf_invalid_context,
 683                         .attr_groups  = attr_groups,
 684                         .event_init   = ddr_perf_event_init,
 685                         .add          = ddr_perf_event_add,
 686                         .del          = ddr_perf_event_del,
 687                         .start        = ddr_perf_event_start,
 688                         .stop         = ddr_perf_event_stop,
 689                         .read         = ddr_perf_event_update,
 690                         .pmu_enable   = ddr_perf_pmu_enable,
 691                         .pmu_disable  = ddr_perf_pmu_disable,
 692                 },
 693                 .base = base,
 694                 .dev = dev,
 695         };
 696 }
 697
 698 static irqreturn_t ddr_perf_irq_handler(int irq, void *p)
 699 {
 700         struct ddr_pmu *pmu = (struct ddr_pmu *)p;
 701         struct perf_event *event;
 702         int i;
 703
 704         /*
 705          * Counters can generate an interrupt on an overflow when msb of a
 706          * counter changes from 0 to 1. For the interrupt to be signalled,
 707          * below condition mush be satisfied:
 708          * PMGC0[PMIE] = 1, PMGC0[FCECE] = 1, PMLCAn[CE] = 1
 709          * When an interrupt is signalled, PMGC0[FAC] is set by hardware and
 710          * all of the registers are frozen.
 711          * Software can clear the interrupt condition by resetting the performance
 712          * monitor and clearing the most significant bit of the counter that
 713          * generate the overflow.
 714          */
 715         for (i = 0; i < NUM_COUNTERS; i++) {
 716                 if (!pmu->events[i])
 717                         continue;
 718
 719                 event = pmu->events[i];
 720
 721                 ddr_perf_event_update(event);
 722         }
 723
 724         ddr_perf_counter_global_config(pmu, true);
 725
 726         return IRQ_HANDLED;
 727 }
 728
 729 static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
 730 {
 731         struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);
 732         int target;
 733
 734         if (cpu != pmu->cpu)
 735                 return 0;
 736
 737         target = cpumask_any_but(cpu_online_mask, cpu);
 738         if (target >= nr_cpu_ids)
 739                 return 0;
 740
 741         perf_pmu_migrate_context(&pmu->pmu, cpu, target);
 742         pmu->cpu = target;
 743
 744         WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)));
 745
 746         return 0;
 747 }
 748
 749 static int ddr_perf_probe(struct platform_device *pdev)
 750 {
 751         struct ddr_pmu *pmu;
 752         void __iomem *base;
 753         int ret, irq;
 754         char *name;
 755
 756         base = devm_platform_ioremap_resource(pdev, 0);
 757         if (IS_ERR(base))
 758                 return PTR_ERR(base);
 759
 760         pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);
 761         if (!pmu)
 762                 return -ENOMEM;
 763
 764         ddr_perf_init(pmu, base, &pdev->dev);
 765
 766         pmu->devtype_data = of_device_get_match_data(&pdev->dev);
 767
 768         platform_set_drvdata(pdev, pmu);
 769
 770         pmu->id = ida_alloc(&ddr_ida, GFP_KERNEL);
 771         name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", pmu->id);
 772         if (!name) {
 773                 ret = -ENOMEM;
 774                 goto format_string_err;
 775         }
 776
 777         pmu->cpu = raw_smp_processor_id();
 778         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DDR_CPUHP_CB_NAME,
 779                                       NULL, ddr_perf_offline_cpu);
 780         if (ret < 0) {
 781                 dev_err(&pdev->dev, "Failed to add callbacks for multi state\n");
 782                 goto cpuhp_state_err;
 783         }
 784         pmu->cpuhp_state = ret;
 785
 786         /* Register the pmu instance for cpu hotplug */
 787         ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 788         if (ret) {
 789                 dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
 790                 goto cpuhp_instance_err;
 791         }
 792
 793         /* Request irq */
 794         irq = platform_get_irq(pdev, 0);
 795         if (irq < 0) {
 796                 ret = irq;
 797                 goto ddr_perf_err;
 798         }
 799
 800         ret = devm_request_irq(&pdev->dev, irq, ddr_perf_irq_handler,
 801                                IRQF_NOBALANCING | IRQF_NO_THREAD,
 802                                DDR_CPUHP_CB_NAME, pmu);
 803         if (ret < 0) {
 804                 dev_err(&pdev->dev, "Request irq failed: %d", ret);
 805                 goto ddr_perf_err;
 806         }
 807
 808         pmu->irq = irq;
 809         ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu));
 810         if (ret) {
 811                 dev_err(pmu->dev, "Failed to set interrupt affinity\n");
 812                 goto ddr_perf_err;
 813         }
 814
 815         ret = perf_pmu_register(&pmu->pmu, name, -1);
 816         if (ret)
 817                 goto ddr_perf_err;
 818
 819         return 0;
 820
 821 ddr_perf_err:
 822         cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 823 cpuhp_instance_err:
 824         cpuhp_remove_multi_state(pmu->cpuhp_state);
 825 cpuhp_state_err:
 826 format_string_err:
 827         ida_free(&ddr_ida, pmu->id);
 828         dev_warn(&pdev->dev, "i.MX9 DDR Perf PMU failed (%d), disabled\n", ret);
 829         return ret;
 830 }
 831
 832 static void ddr_perf_remove(struct platform_device *pdev)
 833 {
 834         struct ddr_pmu *pmu = platform_get_drvdata(pdev);
 835
 836         cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 837         cpuhp_remove_multi_state(pmu->cpuhp_state);
 838
 839         perf_pmu_unregister(&pmu->pmu);
 840
 841         ida_free(&ddr_ida, pmu->id);
 842 }
 843
 844 static struct platform_driver imx_ddr_pmu_driver = {
 845         .driver         = {
 846                 .name                = "imx9-ddr-pmu",
 847                 .of_match_table      = imx_ddr_pmu_dt_ids,
 848                 .suppress_bind_attrs = true,
 849         },
 850         .probe          = ddr_perf_probe,
 851         .remove_new     = ddr_perf_remove,
 852 };
 853 module_platform_driver(imx_ddr_pmu_driver);
 854
 855 MODULE_AUTHOR("Xu Yang <[email protected]>");
 856 MODULE_LICENSE("GPL v2");
 857 MODULE_DESCRIPTION("DDRC PerfMon for i.MX9 SoCs");