drivers/perf/amlogic/meson_ddr_pmu_core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2022 Amlogic, Inc. All rights reserved.
   4  */
   5
   6 #include <linux/bitfield.h>
   7 #include <linux/init.h>
   8 #include <linux/irqreturn.h>
   9 #include <linux/kernel.h>
  10 #include <linux/module.h>
  11 #include <linux/of.h>
  12 #include <linux/of_device.h>
  13 #include <linux/of_irq.h>
  14 #include <linux/perf_event.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/printk.h>
  17 #include <linux/sysfs.h>
  18 #include <linux/types.h>
  19
  20 #include <soc/amlogic/meson_ddr_pmu.h>
  21
  22 struct ddr_pmu {
  23         struct pmu pmu;
  24         struct dmc_info info;
  25         struct dmc_counter counters;    /* save counters from hw */
  26         bool pmu_enabled;
  27         struct device *dev;
  28         char *name;
  29         struct hlist_node node;
  30         enum cpuhp_state cpuhp_state;
  31         int cpu;                        /* for cpu hotplug */
  32 };
  33
  34 #define DDR_PERF_DEV_NAME "meson_ddr_bw"
  35 #define MAX_AXI_PORTS_OF_CHANNEL        4       /* A DMC channel can monitor max 4 axi ports */
  36
  37 #define to_ddr_pmu(p)           container_of(p, struct ddr_pmu, pmu)
  38 #define dmc_info_to_pmu(p)      container_of(p, struct ddr_pmu, info)
  39
  40 static void dmc_pmu_enable(struct ddr_pmu *pmu)
  41 {
  42         if (!pmu->pmu_enabled)
  43                 pmu->info.hw_info->enable(&pmu->info);
  44
  45         pmu->pmu_enabled = true;
  46 }
  47
  48 static void dmc_pmu_disable(struct ddr_pmu *pmu)
  49 {
  50         if (pmu->pmu_enabled)
  51                 pmu->info.hw_info->disable(&pmu->info);
  52
  53         pmu->pmu_enabled = false;
  54 }
  55
  56 static void meson_ddr_set_axi_filter(struct perf_event *event, u8 axi_id)
  57 {
  58         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
  59         int chann;
  60
  61         if (event->attr.config > ALL_CHAN_COUNTER_ID &&
  62             event->attr.config < COUNTER_MAX_ID) {
  63                 chann = event->attr.config - CHAN1_COUNTER_ID;
  64
  65                 pmu->info.hw_info->set_axi_filter(&pmu->info, axi_id, chann);
  66         }
  67 }
  68
  69 static void ddr_cnt_addition(struct dmc_counter *sum,
  70                              struct dmc_counter *add1,
  71                              struct dmc_counter *add2,
  72                              int chann_nr)
  73 {
  74         int i;
  75         u64 cnt1, cnt2;
  76
  77         sum->all_cnt = add1->all_cnt + add2->all_cnt;
  78         sum->all_req = add1->all_req + add2->all_req;
  79         for (i = 0; i < chann_nr; i++) {
  80                 cnt1 = add1->channel_cnt[i];
  81                 cnt2 = add2->channel_cnt[i];
  82
  83                 sum->channel_cnt[i] = cnt1 + cnt2;
  84         }
  85 }
  86
  87 static void meson_ddr_perf_event_update(struct perf_event *event)
  88 {
  89         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
  90         u64 new_raw_count = 0;
  91         struct dmc_counter dc = {0}, sum_dc = {0};
  92         int idx;
  93         int chann_nr = pmu->info.hw_info->chann_nr;
  94
  95         /* get the remain counters in register. */
  96         pmu->info.hw_info->get_counters(&pmu->info, &dc);
  97
  98         ddr_cnt_addition(&sum_dc, &pmu->counters, &dc, chann_nr);
  99
 100         switch (event->attr.config) {
 101         case ALL_CHAN_COUNTER_ID:
 102                 new_raw_count = sum_dc.all_cnt;
 103                 break;
 104         case CHAN1_COUNTER_ID:
 105         case CHAN2_COUNTER_ID:
 106         case CHAN3_COUNTER_ID:
 107         case CHAN4_COUNTER_ID:
 108         case CHAN5_COUNTER_ID:
 109         case CHAN6_COUNTER_ID:
 110         case CHAN7_COUNTER_ID:
 111         case CHAN8_COUNTER_ID:
 112                 idx = event->attr.config - CHAN1_COUNTER_ID;
 113                 new_raw_count = sum_dc.channel_cnt[idx];
 114                 break;
 115         }
 116
 117         local64_set(&event->count, new_raw_count);
 118 }
 119
 120 static int meson_ddr_perf_event_init(struct perf_event *event)
 121 {
 122         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 123         u64 config1 = event->attr.config1;
 124         u64 config2 = event->attr.config2;
 125
 126         if (event->attr.type != event->pmu->type)
 127                 return -ENOENT;
 128
 129         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 130                 return -EOPNOTSUPP;
 131
 132         if (event->cpu < 0)
 133                 return -EOPNOTSUPP;
 134
 135         /* check if the number of parameters is too much */
 136         if (event->attr.config != ALL_CHAN_COUNTER_ID &&
 137             hweight64(config1) + hweight64(config2) > MAX_AXI_PORTS_OF_CHANNEL)
 138                 return -EOPNOTSUPP;
 139
 140         event->cpu = pmu->cpu;
 141
 142         return 0;
 143 }
 144
 145 static void meson_ddr_perf_event_start(struct perf_event *event, int flags)
 146 {
 147         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 148
 149         memset(&pmu->counters, 0, sizeof(pmu->counters));
 150         dmc_pmu_enable(pmu);
 151 }
 152
 153 static int meson_ddr_perf_event_add(struct perf_event *event, int flags)
 154 {
 155         u64 config1 = event->attr.config1;
 156         u64 config2 = event->attr.config2;
 157         int i;
 158
 159         for_each_set_bit(i,
 160                          (const unsigned long *)&config1,
 161                          BITS_PER_TYPE(config1))
 162                 meson_ddr_set_axi_filter(event, i);
 163
 164         for_each_set_bit(i,
 165                          (const unsigned long *)&config2,
 166                          BITS_PER_TYPE(config2))
 167                 meson_ddr_set_axi_filter(event, i + 64);
 168
 169         if (flags & PERF_EF_START)
 170                 meson_ddr_perf_event_start(event, flags);
 171
 172         return 0;
 173 }
 174
 175 static void meson_ddr_perf_event_stop(struct perf_event *event, int flags)
 176 {
 177         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 178
 179         if (flags & PERF_EF_UPDATE)
 180                 meson_ddr_perf_event_update(event);
 181
 182         dmc_pmu_disable(pmu);
 183 }
 184
 185 static void meson_ddr_perf_event_del(struct perf_event *event, int flags)
 186 {
 187         meson_ddr_perf_event_stop(event, PERF_EF_UPDATE);
 188 }
 189
 190 static ssize_t meson_ddr_perf_cpumask_show(struct device *dev,
 191                                            struct device_attribute *attr,
 192                                            char *buf)
 193 {
 194         struct ddr_pmu *pmu = dev_get_drvdata(dev);
 195
 196         return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
 197 }
 198
 199 static struct device_attribute meson_ddr_perf_cpumask_attr =
 200 __ATTR(cpumask, 0444, meson_ddr_perf_cpumask_show, NULL);
 201
 202 static struct attribute *meson_ddr_perf_cpumask_attrs[] = {
 203         &meson_ddr_perf_cpumask_attr.attr,
 204         NULL,
 205 };
 206
 207 static const struct attribute_group ddr_perf_cpumask_attr_group = {
 208         .attrs = meson_ddr_perf_cpumask_attrs,
 209 };
 210
 211 static ssize_t
 212 pmu_event_show(struct device *dev, struct device_attribute *attr,
 213                char *page)
 214 {
 215         struct perf_pmu_events_attr *pmu_attr;
 216
 217         pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
 218         return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
 219 }
 220
 221 static ssize_t
 222 event_show_unit(struct device *dev, struct device_attribute *attr,
 223                 char *page)
 224 {
 225         return sysfs_emit(page, "MB\n");
 226 }
 227
 228 static ssize_t
 229 event_show_scale(struct device *dev, struct device_attribute *attr,
 230                  char *page)
 231 {
 232         /* one count = 16byte = 1.52587890625e-05 MB */
 233         return sysfs_emit(page, "1.52587890625e-05\n");
 234 }
 235
 236 #define AML_DDR_PMU_EVENT_ATTR(_name, _id)                              \
 237 {                                                                       \
 238         .attr = __ATTR(_name, 0444, pmu_event_show, NULL),              \
 239         .id = _id,                                                      \
 240 }
 241
 242 #define AML_DDR_PMU_EVENT_UNIT_ATTR(_name)                              \
 243         __ATTR(_name.unit, 0444, event_show_unit, NULL)
 244
 245 #define AML_DDR_PMU_EVENT_SCALE_ATTR(_name)                             \
 246         __ATTR(_name.scale, 0444, event_show_scale, NULL)
 247
 248 static struct device_attribute event_unit_attrs[] = {
 249         AML_DDR_PMU_EVENT_UNIT_ATTR(total_rw_bytes),
 250         AML_DDR_PMU_EVENT_UNIT_ATTR(chan_1_rw_bytes),
 251         AML_DDR_PMU_EVENT_UNIT_ATTR(chan_2_rw_bytes),
 252         AML_DDR_PMU_EVENT_UNIT_ATTR(chan_3_rw_bytes),
 253         AML_DDR_PMU_EVENT_UNIT_ATTR(chan_4_rw_bytes),
 254         AML_DDR_PMU_EVENT_UNIT_ATTR(chan_5_rw_bytes),
 255         AML_DDR_PMU_EVENT_UNIT_ATTR(chan_6_rw_bytes),
 256         AML_DDR_PMU_EVENT_UNIT_ATTR(chan_7_rw_bytes),
 257         AML_DDR_PMU_EVENT_UNIT_ATTR(chan_8_rw_bytes),
 258 };
 259
 260 static struct device_attribute event_scale_attrs[] = {
 261         AML_DDR_PMU_EVENT_SCALE_ATTR(total_rw_bytes),
 262         AML_DDR_PMU_EVENT_SCALE_ATTR(chan_1_rw_bytes),
 263         AML_DDR_PMU_EVENT_SCALE_ATTR(chan_2_rw_bytes),
 264         AML_DDR_PMU_EVENT_SCALE_ATTR(chan_3_rw_bytes),
 265         AML_DDR_PMU_EVENT_SCALE_ATTR(chan_4_rw_bytes),
 266         AML_DDR_PMU_EVENT_SCALE_ATTR(chan_5_rw_bytes),
 267         AML_DDR_PMU_EVENT_SCALE_ATTR(chan_6_rw_bytes),
 268         AML_DDR_PMU_EVENT_SCALE_ATTR(chan_7_rw_bytes),
 269         AML_DDR_PMU_EVENT_SCALE_ATTR(chan_8_rw_bytes),
 270 };
 271
 272 static struct perf_pmu_events_attr event_attrs[] = {
 273         AML_DDR_PMU_EVENT_ATTR(total_rw_bytes, ALL_CHAN_COUNTER_ID),
 274         AML_DDR_PMU_EVENT_ATTR(chan_1_rw_bytes, CHAN1_COUNTER_ID),
 275         AML_DDR_PMU_EVENT_ATTR(chan_2_rw_bytes, CHAN2_COUNTER_ID),
 276         AML_DDR_PMU_EVENT_ATTR(chan_3_rw_bytes, CHAN3_COUNTER_ID),
 277         AML_DDR_PMU_EVENT_ATTR(chan_4_rw_bytes, CHAN4_COUNTER_ID),
 278         AML_DDR_PMU_EVENT_ATTR(chan_5_rw_bytes, CHAN5_COUNTER_ID),
 279         AML_DDR_PMU_EVENT_ATTR(chan_6_rw_bytes, CHAN6_COUNTER_ID),
 280         AML_DDR_PMU_EVENT_ATTR(chan_7_rw_bytes, CHAN7_COUNTER_ID),
 281         AML_DDR_PMU_EVENT_ATTR(chan_8_rw_bytes, CHAN8_COUNTER_ID),
 282 };
 283
 284 /* three attrs are combined an event */
 285 static struct attribute *ddr_perf_events_attrs[COUNTER_MAX_ID * 3];
 286
 287 static struct attribute_group ddr_perf_events_attr_group = {
 288         .name = "events",
 289         .attrs = ddr_perf_events_attrs,
 290 };
 291
 292 static umode_t meson_ddr_perf_format_attr_visible(struct kobject *kobj,
 293                                                   struct attribute *attr,
 294                                                   int n)
 295 {
 296         struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
 297         struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
 298         const u64 *capability = ddr_pmu->info.hw_info->capability;
 299         struct device_attribute *dev_attr;
 300         int id;
 301         char value[20]; // config1:xxx, 20 is enough
 302
 303         dev_attr = container_of(attr, struct device_attribute, attr);
 304         dev_attr->show(NULL, NULL, value);
 305
 306         if (sscanf(value, "config1:%d", &id) == 1)
 307                 return capability[0] & (1ULL << id) ? attr->mode : 0;
 308
 309         if (sscanf(value, "config2:%d", &id) == 1)
 310                 return capability[1] & (1ULL << id) ? attr->mode : 0;
 311
 312         return attr->mode;
 313 }
 314
 315 static struct attribute_group ddr_perf_format_attr_group = {
 316         .name = "format",
 317         .is_visible = meson_ddr_perf_format_attr_visible,
 318 };
 319
 320 static ssize_t meson_ddr_perf_identifier_show(struct device *dev,
 321                                               struct device_attribute *attr,
 322                                               char *page)
 323 {
 324         struct ddr_pmu *pmu = dev_get_drvdata(dev);
 325
 326         return sysfs_emit(page, "%s\n", pmu->name);
 327 }
 328
 329 static struct device_attribute meson_ddr_perf_identifier_attr =
 330 __ATTR(identifier, 0444, meson_ddr_perf_identifier_show, NULL);
 331
 332 static struct attribute *meson_ddr_perf_identifier_attrs[] = {
 333         &meson_ddr_perf_identifier_attr.attr,
 334         NULL,
 335 };
 336
 337 static const struct attribute_group ddr_perf_identifier_attr_group = {
 338         .attrs = meson_ddr_perf_identifier_attrs,
 339 };
 340
 341 static const struct attribute_group *attr_groups[] = {
 342         &ddr_perf_events_attr_group,
 343         &ddr_perf_format_attr_group,
 344         &ddr_perf_cpumask_attr_group,
 345         &ddr_perf_identifier_attr_group,
 346         NULL,
 347 };
 348
 349 static irqreturn_t dmc_irq_handler(int irq, void *dev_id)
 350 {
 351         struct dmc_info *info = dev_id;
 352         struct ddr_pmu *pmu;
 353         struct dmc_counter counters, *sum_cnter;
 354         int i;
 355
 356         pmu = dmc_info_to_pmu(info);
 357
 358         if (info->hw_info->irq_handler(info, &counters) != 0)
 359                 goto out;
 360
 361         sum_cnter = &pmu->counters;
 362         sum_cnter->all_cnt += counters.all_cnt;
 363         sum_cnter->all_req += counters.all_req;
 364
 365         for (i = 0; i < pmu->info.hw_info->chann_nr; i++)
 366                 sum_cnter->channel_cnt[i] += counters.channel_cnt[i];
 367
 368         if (pmu->pmu_enabled)
 369                 /*
 370                  * the timer interrupt only supprt
 371                  * one shot mode, we have to re-enable
 372                  * it in ISR to support continue mode.
 373                  */
 374                 info->hw_info->enable(info);
 375
 376         dev_dbg(pmu->dev, "counts: %llu %llu %llu, %llu, %llu, %llu\t\t"
 377                         "sum: %llu %llu %llu, %llu, %llu, %llu\n",
 378                         counters.all_req,
 379                         counters.all_cnt,
 380                         counters.channel_cnt[0],
 381                         counters.channel_cnt[1],
 382                         counters.channel_cnt[2],
 383                         counters.channel_cnt[3],
 384
 385                         pmu->counters.all_req,
 386                         pmu->counters.all_cnt,
 387                         pmu->counters.channel_cnt[0],
 388                         pmu->counters.channel_cnt[1],
 389                         pmu->counters.channel_cnt[2],
 390                         pmu->counters.channel_cnt[3]);
 391 out:
 392         return IRQ_HANDLED;
 393 }
 394
 395 static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
 396 {
 397         struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);
 398         int target;
 399
 400         if (cpu != pmu->cpu)
 401                 return 0;
 402
 403         target = cpumask_any_but(cpu_online_mask, cpu);
 404         if (target >= nr_cpu_ids)
 405                 return 0;
 406
 407         perf_pmu_migrate_context(&pmu->pmu, cpu, target);
 408         pmu->cpu = target;
 409
 410         WARN_ON(irq_set_affinity(pmu->info.irq_num, cpumask_of(pmu->cpu)));
 411
 412         return 0;
 413 }
 414
 415 static void fill_event_attr(struct ddr_pmu *pmu)
 416 {
 417         int i, j, k;
 418         struct attribute **dst = ddr_perf_events_attrs;
 419
 420         j = 0;
 421         k = 0;
 422
 423         /* fill ALL_CHAN_COUNTER_ID event */
 424         dst[j++] = &event_attrs[k].attr.attr;
 425         dst[j++] = &event_unit_attrs[k].attr;
 426         dst[j++] = &event_scale_attrs[k].attr;
 427
 428         k++;
 429
 430         /* fill each channel event */
 431         for (i = 0; i < pmu->info.hw_info->chann_nr; i++, k++) {
 432                 dst[j++] = &event_attrs[k].attr.attr;
 433                 dst[j++] = &event_unit_attrs[k].attr;
 434                 dst[j++] = &event_scale_attrs[k].attr;
 435         }
 436
 437         dst[j] = NULL; /* mark end */
 438 }
 439
 440 static void fmt_attr_fill(struct attribute **fmt_attr)
 441 {
 442         ddr_perf_format_attr_group.attrs = fmt_attr;
 443 }
 444
 445 static int ddr_pmu_parse_dt(struct platform_device *pdev,
 446                             struct dmc_info *info)
 447 {
 448         void __iomem *base;
 449         int i, ret;
 450
 451         info->hw_info = of_device_get_match_data(&pdev->dev);
 452
 453         for (i = 0; i < info->hw_info->dmc_nr; i++) {
 454                 /* resource 0 for ddr register base */
 455                 base = devm_platform_ioremap_resource(pdev, i);
 456                 if (IS_ERR(base))
 457                         return PTR_ERR(base);
 458
 459                 info->ddr_reg[i] = base;
 460         }
 461
 462         /* resource i for pll register base */
 463         base = devm_platform_ioremap_resource(pdev, i);
 464         if (IS_ERR(base))
 465                 return PTR_ERR(base);
 466
 467         info->pll_reg = base;
 468
 469         ret = platform_get_irq(pdev, 0);
 470         if (ret < 0)
 471                 return ret;
 472
 473         info->irq_num = ret;
 474
 475         ret = devm_request_irq(&pdev->dev, info->irq_num, dmc_irq_handler,
 476                                IRQF_NOBALANCING, dev_name(&pdev->dev),
 477                                (void *)info);
 478         if (ret < 0)
 479                 return ret;
 480
 481         return 0;
 482 }
 483
 484 int meson_ddr_pmu_create(struct platform_device *pdev)
 485 {
 486         int ret;
 487         char *name;
 488         struct ddr_pmu *pmu;
 489
 490         pmu = devm_kzalloc(&pdev->dev, sizeof(struct ddr_pmu), GFP_KERNEL);
 491         if (!pmu)
 492                 return -ENOMEM;
 493
 494         *pmu = (struct ddr_pmu) {
 495                 .pmu = {
 496                         .module         = THIS_MODULE,
 497                         .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
 498                         .task_ctx_nr    = perf_invalid_context,
 499                         .attr_groups    = attr_groups,
 500                         .event_init     = meson_ddr_perf_event_init,
 501                         .add            = meson_ddr_perf_event_add,
 502                         .del            = meson_ddr_perf_event_del,
 503                         .start          = meson_ddr_perf_event_start,
 504                         .stop           = meson_ddr_perf_event_stop,
 505                         .read           = meson_ddr_perf_event_update,
 506                 },
 507         };
 508
 509         ret = ddr_pmu_parse_dt(pdev, &pmu->info);
 510         if (ret < 0)
 511                 return ret;
 512
 513         fmt_attr_fill(pmu->info.hw_info->fmt_attr);
 514
 515         pmu->cpu = smp_processor_id();
 516
 517         name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME);
 518         if (!name)
 519                 return -ENOMEM;
 520
 521         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, name, NULL,
 522                                       ddr_perf_offline_cpu);
 523         if (ret < 0)
 524                 return ret;
 525
 526         pmu->cpuhp_state = ret;
 527
 528         /* Register the pmu instance for cpu hotplug */
 529         ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 530         if (ret)
 531                 goto cpuhp_instance_err;
 532
 533         fill_event_attr(pmu);
 534
 535         ret = perf_pmu_register(&pmu->pmu, name, -1);
 536         if (ret)
 537                 goto pmu_register_err;
 538
 539         pmu->name = name;
 540         pmu->dev = &pdev->dev;
 541         pmu->pmu_enabled = false;
 542
 543         platform_set_drvdata(pdev, pmu);
 544
 545         return 0;
 546
 547 pmu_register_err:
 548         cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 549
 550 cpuhp_instance_err:
 551         cpuhp_remove_state(pmu->cpuhp_state);
 552
 553         return ret;
 554 }
 555
 556 int meson_ddr_pmu_remove(struct platform_device *pdev)
 557 {
 558         struct ddr_pmu *pmu = platform_get_drvdata(pdev);
 559
 560         perf_pmu_unregister(&pmu->pmu);
 561         cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 562         cpuhp_remove_state(pmu->cpuhp_state);
 563
 564         return 0;
 565 }