drivers/perf/alibaba_uncore_drw_pmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Alibaba DDR Sub-System Driveway PMU driver
   4  *
   5  * Copyright (C) 2022 Alibaba Inc
   6  */
   7
   8 #define ALI_DRW_PMUNAME         "ali_drw"
   9 #define ALI_DRW_DRVNAME         ALI_DRW_PMUNAME "_pmu"
  10 #define pr_fmt(fmt)             ALI_DRW_DRVNAME ": " fmt
  11
  12 #include <linux/acpi.h>
  13 #include <linux/bitfield.h>
  14 #include <linux/bitmap.h>
  15 #include <linux/bitops.h>
  16 #include <linux/cpuhotplug.h>
  17 #include <linux/cpumask.h>
  18 #include <linux/device.h>
  19 #include <linux/errno.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/irq.h>
  22 #include <linux/kernel.h>
  23 #include <linux/list.h>
  24 #include <linux/module.h>
  25 #include <linux/mutex.h>
  26 #include <linux/perf_event.h>
  27 #include <linux/platform_device.h>
  28 #include <linux/printk.h>
  29 #include <linux/rculist.h>
  30 #include <linux/refcount.h>
  31
  32
  33 #define ALI_DRW_PMU_COMMON_MAX_COUNTERS                 16
  34 #define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE        19
  35
  36 #define ALI_DRW_PMU_PA_SHIFT                    12
  37 #define ALI_DRW_PMU_CNT_INIT                    0x00000000
  38 #define ALI_DRW_CNT_MAX_PERIOD                  0xffffffff
  39 #define ALI_DRW_PMU_CYCLE_EVT_ID                0x80
  40
  41 #define ALI_DRW_PMU_CNT_CTRL                    0xC00
  42 #define ALI_DRW_PMU_CNT_RST                     BIT(2)
  43 #define ALI_DRW_PMU_CNT_STOP                    BIT(1)
  44 #define ALI_DRW_PMU_CNT_START                   BIT(0)
  45
  46 #define ALI_DRW_PMU_CNT_STATE                   0xC04
  47 #define ALI_DRW_PMU_TEST_CTRL                   0xC08
  48 #define ALI_DRW_PMU_CNT_PRELOAD                 0xC0C
  49
  50 #define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK         GENMASK(23, 0)
  51 #define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK          GENMASK(31, 0)
  52 #define ALI_DRW_PMU_CYCLE_CNT_HIGH              0xC10
  53 #define ALI_DRW_PMU_CYCLE_CNT_LOW               0xC14
  54
  55 /* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */
  56 #define ALI_DRW_PMU_EVENT_SEL0                  0xC68
  57 /* counter 0-3 use sel0, counter 4-7 use sel1...*/
  58 #define ALI_DRW_PMU_EVENT_SELn(n) \
  59         (ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4)
  60 #define ALI_DRW_PMCOM_CNT_EN                    BIT(7)
  61 #define ALI_DRW_PMCOM_CNT_EVENT_MASK            GENMASK(5, 0)
  62 #define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \
  63         (8 * (n % 4))
  64
  65 /* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */
  66 #define ALI_DRW_PMU_COMMON_COUNTER0             0xC78
  67 #define ALI_DRW_PMU_COMMON_COUNTERn(n) \
  68         (ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n))
  69
  70 #define ALI_DRW_PMU_OV_INTR_ENABLE_CTL          0xCB8
  71 #define ALI_DRW_PMU_OV_INTR_DISABLE_CTL         0xCBC
  72 #define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS       0xCC0
  73 #define ALI_DRW_PMU_OV_INTR_CLR                 0xCC4
  74 #define ALI_DRW_PMU_OV_INTR_STATUS              0xCC8
  75 #define ALI_DRW_PMCOM_CNT_OV_INTR_MASK          GENMASK(23, 8)
  76 #define ALI_DRW_PMBW_CNT_OV_INTR_MASK           GENMASK(7, 0)
  77 #define ALI_DRW_PMU_OV_INTR_MASK                GENMASK_ULL(63, 0)
  78
  79 static int ali_drw_cpuhp_state_num;
  80
  81 static LIST_HEAD(ali_drw_pmu_irqs);
  82 static DEFINE_MUTEX(ali_drw_pmu_irqs_lock);
  83
  84 struct ali_drw_pmu_irq {
  85         struct hlist_node node;
  86         struct list_head irqs_node;
  87         struct list_head pmus_node;
  88         int irq_num;
  89         int cpu;
  90         refcount_t refcount;
  91 };
  92
  93 struct ali_drw_pmu {
  94         void __iomem *cfg_base;
  95         struct device *dev;
  96
  97         struct list_head pmus_node;
  98         struct ali_drw_pmu_irq *irq;
  99         int irq_num;
 100         int cpu;
 101         DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS);
 102         struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
 103         int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
 104
 105         struct pmu pmu;
 106 };
 107
 108 #define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu))
 109
 110 #define DRW_CONFIG_EVENTID              GENMASK(7, 0)
 111 #define GET_DRW_EVENTID(event)  FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config)
 112
 113 static ssize_t ali_drw_pmu_format_show(struct device *dev,
 114                                 struct device_attribute *attr, char *buf)
 115 {
 116         struct dev_ext_attribute *eattr;
 117
 118         eattr = container_of(attr, struct dev_ext_attribute, attr);
 119
 120         return sprintf(buf, "%s\n", (char *)eattr->var);
 121 }
 122
 123 /*
 124  * PMU event attributes
 125  */
 126 static ssize_t ali_drw_pmu_event_show(struct device *dev,
 127                                struct device_attribute *attr, char *page)
 128 {
 129         struct dev_ext_attribute *eattr;
 130
 131         eattr = container_of(attr, struct dev_ext_attribute, attr);
 132
 133         return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
 134 }
 135
 136 #define ALI_DRW_PMU_ATTR(_name, _func, _config)                            \
 137                 (&((struct dev_ext_attribute[]) {                               \
 138                                 { __ATTR(_name, 0444, _func, NULL), (void *)_config }   \
 139                 })[0].attr.attr)
 140
 141 #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config)            \
 142         ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config)
 143 #define ALI_DRW_PMU_EVENT_ATTR(_name, _config)             \
 144         ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config)
 145
 146 static struct attribute *ali_drw_pmu_events_attrs[] = {
 147         ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr,                    0x0),
 148         ALI_DRW_PMU_EVENT_ATTR(hif_wr,                          0x1),
 149         ALI_DRW_PMU_EVENT_ATTR(hif_rd,                          0x2),
 150         ALI_DRW_PMU_EVENT_ATTR(hif_rmw,                         0x3),
 151         ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd,                   0x4),
 152         ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles,              0x7),
 153         ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles,              0x8),
 154         ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical,          0x9),
 155         ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical,          0xA),
 156         ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical,           0xB),
 157         ALI_DRW_PMU_EVENT_ATTR(op_is_activate,                  0xC),
 158         ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr,                  0xD),
 159         ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate,               0xE),
 160         ALI_DRW_PMU_EVENT_ATTR(op_is_rd,                        0xF),
 161         ALI_DRW_PMU_EVENT_ATTR(op_is_wr,                        0x10),
 162         ALI_DRW_PMU_EVENT_ATTR(op_is_mwr,                       0x11),
 163         ALI_DRW_PMU_EVENT_ATTR(op_is_precharge,                 0x12),
 164         ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr,              0x13),
 165         ALI_DRW_PMU_EVENT_ATTR(precharge_for_other,             0x14),
 166         ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions,                0x15),
 167         ALI_DRW_PMU_EVENT_ATTR(write_combine,                   0x16),
 168         ALI_DRW_PMU_EVENT_ATTR(war_hazard,                      0x17),
 169         ALI_DRW_PMU_EVENT_ATTR(raw_hazard,                      0x18),
 170         ALI_DRW_PMU_EVENT_ATTR(waw_hazard,                      0x19),
 171         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0,         0x1A),
 172         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1,         0x1B),
 173         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2,         0x1C),
 174         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3,         0x1D),
 175         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0,       0x1E),
 176         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1,       0x1F),
 177         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2,       0x20),
 178         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3,       0x21),
 179         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0,                0x26),
 180         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1,                0x27),
 181         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2,                0x28),
 182         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3,                0x29),
 183         ALI_DRW_PMU_EVENT_ATTR(op_is_refresh,                   0x2A),
 184         ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref,                  0x2B),
 185         ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode,                 0x2D),
 186         ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl,                      0x2E),
 187         ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30),
 188         ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31),
 189         ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc,                0x34),
 190         ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr,                0x35),
 191         ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr,                   0x36),
 192         ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart,                   0x37),
 193         ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch,                   0x38),
 194         ALI_DRW_PMU_EVENT_ATTR(chi_txreq,                       0x39),
 195         ALI_DRW_PMU_EVENT_ATTR(chi_txdat,                       0x3A),
 196         ALI_DRW_PMU_EVENT_ATTR(chi_rxdat,                       0x3B),
 197         ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp,                       0x3C),
 198         ALI_DRW_PMU_EVENT_ATTR(tsz_vio,                         0x3D),
 199         ALI_DRW_PMU_EVENT_ATTR(cycle,                           0x80),
 200         NULL,
 201 };
 202
 203 static struct attribute_group ali_drw_pmu_events_attr_group = {
 204         .name = "events",
 205         .attrs = ali_drw_pmu_events_attrs,
 206 };
 207
 208 static struct attribute *ali_drw_pmu_format_attr[] = {
 209         ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"),
 210         NULL,
 211 };
 212
 213 static const struct attribute_group ali_drw_pmu_format_group = {
 214         .name = "format",
 215         .attrs = ali_drw_pmu_format_attr,
 216 };
 217
 218 static ssize_t ali_drw_pmu_cpumask_show(struct device *dev,
 219                                         struct device_attribute *attr,
 220                                         char *buf)
 221 {
 222         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev));
 223
 224         return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu));
 225 }
 226
 227 static struct device_attribute ali_drw_pmu_cpumask_attr =
 228                 __ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL);
 229
 230 static struct attribute *ali_drw_pmu_cpumask_attrs[] = {
 231         &ali_drw_pmu_cpumask_attr.attr,
 232         NULL,
 233 };
 234
 235 static const struct attribute_group ali_drw_pmu_cpumask_attr_group = {
 236         .attrs = ali_drw_pmu_cpumask_attrs,
 237 };
 238
 239 static umode_t ali_drw_pmu_identifier_attr_visible(struct kobject *kobj,
 240                                                 struct attribute *attr, int n)
 241 {
 242         return attr->mode;
 243 }
 244
 245 static DEVICE_STRING_ATTR_RO(ali_drw_pmu_identifier, 0444, "ali_drw_pmu");
 246
 247 static struct attribute *ali_drw_pmu_identifier_attrs[] = {
 248         &dev_attr_ali_drw_pmu_identifier.attr.attr,
 249         NULL
 250 };
 251
 252 static const struct attribute_group ali_drw_pmu_identifier_attr_group = {
 253         .attrs = ali_drw_pmu_identifier_attrs,
 254         .is_visible = ali_drw_pmu_identifier_attr_visible
 255 };
 256
 257 static const struct attribute_group *ali_drw_pmu_attr_groups[] = {
 258         &ali_drw_pmu_events_attr_group,
 259         &ali_drw_pmu_cpumask_attr_group,
 260         &ali_drw_pmu_format_group,
 261         &ali_drw_pmu_identifier_attr_group,
 262         NULL,
 263 };
 264
 265 /* find a counter for event, then in add func, hw.idx will equal to counter */
 266 static int ali_drw_get_counter_idx(struct perf_event *event)
 267 {
 268         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 269         int idx;
 270
 271         for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) {
 272                 if (!test_and_set_bit(idx, drw_pmu->used_mask))
 273                         return idx;
 274         }
 275
 276         /* The counters are all in use. */
 277         return -EBUSY;
 278 }
 279
 280 static u64 ali_drw_pmu_read_counter(struct perf_event *event)
 281 {
 282         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 283         u64 cycle_high, cycle_low;
 284
 285         if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
 286                 cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH);
 287                 cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK;
 288                 cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW);
 289                 cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK;
 290                 return (cycle_high << 32 | cycle_low);
 291         }
 292
 293         return readl(drw_pmu->cfg_base +
 294                      ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx));
 295 }
 296
 297 static void ali_drw_pmu_event_update(struct perf_event *event)
 298 {
 299         struct hw_perf_event *hwc = &event->hw;
 300         u64 delta, prev, now;
 301
 302         do {
 303                 prev = local64_read(&hwc->prev_count);
 304                 now = ali_drw_pmu_read_counter(event);
 305         } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
 306
 307         /* handle overflow. */
 308         delta = now - prev;
 309         if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID)
 310                 delta &= ALI_DRW_PMU_OV_INTR_MASK;
 311         else
 312                 delta &= ALI_DRW_CNT_MAX_PERIOD;
 313         local64_add(delta, &event->count);
 314 }
 315
 316 static void ali_drw_pmu_event_set_period(struct perf_event *event)
 317 {
 318         u64 pre_val;
 319         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 320
 321         /* set a preload counter for test purpose */
 322         writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx,
 323                drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
 324
 325         /* set conunter initial value */
 326         pre_val = ALI_DRW_PMU_CNT_INIT;
 327         writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
 328         local64_set(&event->hw.prev_count, pre_val);
 329
 330         /* set sel mode to zero to start test */
 331         writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
 332 }
 333
 334 static void ali_drw_pmu_enable_counter(struct perf_event *event)
 335 {
 336         u32 val, subval, reg, shift;
 337         int counter = event->hw.idx;
 338         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 339
 340         reg = ALI_DRW_PMU_EVENT_SELn(counter);
 341         val = readl(drw_pmu->cfg_base + reg);
 342         subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) |
 343                  FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]);
 344
 345         shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
 346         val &= ~(GENMASK(7, 0) << shift);
 347         val |= subval << shift;
 348
 349         writel(val, drw_pmu->cfg_base + reg);
 350 }
 351
 352 static void ali_drw_pmu_disable_counter(struct perf_event *event)
 353 {
 354         u32 val, reg, subval, shift;
 355         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 356         int counter = event->hw.idx;
 357
 358         reg = ALI_DRW_PMU_EVENT_SELn(counter);
 359         val = readl(drw_pmu->cfg_base + reg);
 360         subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) |
 361                  FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0);
 362
 363         shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
 364         val &= ~(GENMASK(7, 0) << shift);
 365         val |= subval << shift;
 366
 367         writel(val, drw_pmu->cfg_base + reg);
 368 }
 369
 370 static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data)
 371 {
 372         struct ali_drw_pmu_irq *irq = data;
 373         struct ali_drw_pmu *drw_pmu;
 374         irqreturn_t ret = IRQ_NONE;
 375
 376         rcu_read_lock();
 377         list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) {
 378                 unsigned long status, clr_status;
 379                 struct perf_event *event;
 380                 unsigned int idx;
 381
 382                 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
 383                         event = drw_pmu->events[idx];
 384                         if (!event)
 385                                 continue;
 386                         ali_drw_pmu_disable_counter(event);
 387                 }
 388
 389                 /* common counter intr status */
 390                 status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS);
 391                 status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);
 392                 if (status) {
 393                         for_each_set_bit(idx, &status,
 394                                          ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
 395                                 event = drw_pmu->events[idx];
 396                                 if (WARN_ON_ONCE(!event))
 397                                         continue;
 398                                 ali_drw_pmu_event_update(event);
 399                                 ali_drw_pmu_event_set_period(event);
 400                         }
 401
 402                         /* clear common counter intr status */
 403                         clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);
 404                         writel(clr_status,
 405                                drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
 406                 }
 407
 408                 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
 409                         event = drw_pmu->events[idx];
 410                         if (!event)
 411                                 continue;
 412                         if (!(event->hw.state & PERF_HES_STOPPED))
 413                                 ali_drw_pmu_enable_counter(event);
 414                 }
 415                 if (status)
 416                         ret = IRQ_HANDLED;
 417         }
 418         rcu_read_unlock();
 419         return ret;
 420 }
 421
 422 static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device
 423                                                       *pdev, int irq_num)
 424 {
 425         int ret;
 426         struct ali_drw_pmu_irq *irq;
 427
 428         list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) {
 429                 if (irq->irq_num == irq_num
 430                     && refcount_inc_not_zero(&irq->refcount))
 431                         return irq;
 432         }
 433
 434         irq = kzalloc(sizeof(*irq), GFP_KERNEL);
 435         if (!irq)
 436                 return ERR_PTR(-ENOMEM);
 437
 438         INIT_LIST_HEAD(&irq->pmus_node);
 439
 440         /* Pick one CPU to be the preferred one to use */
 441         irq->cpu = smp_processor_id();
 442         refcount_set(&irq->refcount, 1);
 443
 444         /*
 445          * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same
 446          * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers
 447          * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not
 448          * share with other component.
 449          */
 450         ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr,
 451                                IRQF_SHARED, dev_name(&pdev->dev), irq);
 452         if (ret < 0) {
 453                 dev_err(&pdev->dev,
 454                         "Fail to request IRQ:%d ret:%d\n", irq_num, ret);
 455                 goto out_free;
 456         }
 457
 458         ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu));
 459         if (ret)
 460                 goto out_free;
 461
 462         ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num,
 463                                              &irq->node);
 464         if (ret)
 465                 goto out_free;
 466
 467         irq->irq_num = irq_num;
 468         list_add(&irq->irqs_node, &ali_drw_pmu_irqs);
 469
 470         return irq;
 471
 472 out_free:
 473         kfree(irq);
 474         return ERR_PTR(ret);
 475 }
 476
 477 static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu,
 478                                 struct platform_device *pdev)
 479 {
 480         int irq_num;
 481         struct ali_drw_pmu_irq *irq;
 482
 483         /* Read and init IRQ */
 484         irq_num = platform_get_irq(pdev, 0);
 485         if (irq_num < 0)
 486                 return irq_num;
 487
 488         mutex_lock(&ali_drw_pmu_irqs_lock);
 489         irq = __ali_drw_pmu_init_irq(pdev, irq_num);
 490         mutex_unlock(&ali_drw_pmu_irqs_lock);
 491
 492         if (IS_ERR(irq))
 493                 return PTR_ERR(irq);
 494
 495         drw_pmu->irq = irq;
 496
 497         mutex_lock(&ali_drw_pmu_irqs_lock);
 498         list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node);
 499         mutex_unlock(&ali_drw_pmu_irqs_lock);
 500
 501         return 0;
 502 }
 503
 504 static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu)
 505 {
 506         struct ali_drw_pmu_irq *irq = drw_pmu->irq;
 507
 508         mutex_lock(&ali_drw_pmu_irqs_lock);
 509         list_del_rcu(&drw_pmu->pmus_node);
 510
 511         if (!refcount_dec_and_test(&irq->refcount)) {
 512                 mutex_unlock(&ali_drw_pmu_irqs_lock);
 513                 return;
 514         }
 515
 516         list_del(&irq->irqs_node);
 517         mutex_unlock(&ali_drw_pmu_irqs_lock);
 518
 519         WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL));
 520         cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num,
 521                                             &irq->node);
 522         kfree(irq);
 523 }
 524
 525 static int ali_drw_pmu_event_init(struct perf_event *event)
 526 {
 527         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 528         struct hw_perf_event *hwc = &event->hw;
 529         struct perf_event *sibling;
 530         struct device *dev = drw_pmu->pmu.dev;
 531
 532         if (event->attr.type != event->pmu->type)
 533                 return -ENOENT;
 534
 535         if (is_sampling_event(event)) {
 536                 dev_err(dev, "Sampling not supported!\n");
 537                 return -EOPNOTSUPP;
 538         }
 539
 540         if (event->attach_state & PERF_ATTACH_TASK) {
 541                 dev_err(dev, "Per-task counter cannot allocate!\n");
 542                 return -EOPNOTSUPP;
 543         }
 544
 545         event->cpu = drw_pmu->cpu;
 546         if (event->cpu < 0) {
 547                 dev_err(dev, "Per-task mode not supported!\n");
 548                 return -EOPNOTSUPP;
 549         }
 550
 551         if (event->group_leader != event &&
 552             !is_software_event(event->group_leader)) {
 553                 dev_err(dev, "driveway only allow one event!\n");
 554                 return -EINVAL;
 555         }
 556
 557         for_each_sibling_event(sibling, event->group_leader) {
 558                 if (sibling != event && !is_software_event(sibling)) {
 559                         dev_err(dev, "driveway event not allowed!\n");
 560                         return -EINVAL;
 561                 }
 562         }
 563
 564         /* reset all the pmu counters */
 565         writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 566
 567         hwc->idx = -1;
 568
 569         return 0;
 570 }
 571
 572 static void ali_drw_pmu_start(struct perf_event *event, int flags)
 573 {
 574         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 575
 576         event->hw.state = 0;
 577
 578         if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
 579                 writel(ALI_DRW_PMU_CNT_START,
 580                        drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 581                 return;
 582         }
 583
 584         ali_drw_pmu_event_set_period(event);
 585         if (flags & PERF_EF_RELOAD) {
 586                 unsigned long prev_raw_count =
 587                     local64_read(&event->hw.prev_count);
 588                 writel(prev_raw_count,
 589                        drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
 590         }
 591
 592         ali_drw_pmu_enable_counter(event);
 593
 594         writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 595 }
 596
 597 static void ali_drw_pmu_stop(struct perf_event *event, int flags)
 598 {
 599         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 600
 601         if (event->hw.state & PERF_HES_STOPPED)
 602                 return;
 603
 604         if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID)
 605                 ali_drw_pmu_disable_counter(event);
 606
 607         writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 608
 609         ali_drw_pmu_event_update(event);
 610         event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 611 }
 612
 613 static int ali_drw_pmu_add(struct perf_event *event, int flags)
 614 {
 615         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 616         struct hw_perf_event *hwc = &event->hw;
 617         int idx = -1;
 618         int evtid;
 619
 620         evtid = GET_DRW_EVENTID(event);
 621
 622         if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) {
 623                 idx = ali_drw_get_counter_idx(event);
 624                 if (idx < 0)
 625                         return idx;
 626                 drw_pmu->events[idx] = event;
 627                 drw_pmu->evtids[idx] = evtid;
 628         }
 629         hwc->idx = idx;
 630
 631         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 632
 633         if (flags & PERF_EF_START)
 634                 ali_drw_pmu_start(event, PERF_EF_RELOAD);
 635
 636         /* Propagate our changes to the userspace mapping. */
 637         perf_event_update_userpage(event);
 638
 639         return 0;
 640 }
 641
 642 static void ali_drw_pmu_del(struct perf_event *event, int flags)
 643 {
 644         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 645         struct hw_perf_event *hwc = &event->hw;
 646         int idx = hwc->idx;
 647
 648         ali_drw_pmu_stop(event, PERF_EF_UPDATE);
 649
 650         if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
 651                 drw_pmu->events[idx] = NULL;
 652                 drw_pmu->evtids[idx] = 0;
 653                 clear_bit(idx, drw_pmu->used_mask);
 654         }
 655
 656         perf_event_update_userpage(event);
 657 }
 658
 659 static void ali_drw_pmu_read(struct perf_event *event)
 660 {
 661         ali_drw_pmu_event_update(event);
 662 }
 663
 664 static int ali_drw_pmu_probe(struct platform_device *pdev)
 665 {
 666         struct ali_drw_pmu *drw_pmu;
 667         struct resource *res;
 668         char *name;
 669         int ret;
 670
 671         drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL);
 672         if (!drw_pmu)
 673                 return -ENOMEM;
 674
 675         drw_pmu->dev = &pdev->dev;
 676         platform_set_drvdata(pdev, drw_pmu);
 677
 678         drw_pmu->cfg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 679         if (IS_ERR(drw_pmu->cfg_base))
 680                 return PTR_ERR(drw_pmu->cfg_base);
 681
 682         name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx",
 683                               (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT));
 684         if (!name)
 685                 return -ENOMEM;
 686
 687         writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 688
 689         /* enable the generation of interrupt by all common counters */
 690         writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
 691                drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL);
 692
 693         /* clearing interrupt status */
 694         writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
 695
 696         drw_pmu->cpu = smp_processor_id();
 697
 698         ret = ali_drw_pmu_init_irq(drw_pmu, pdev);
 699         if (ret)
 700                 return ret;
 701
 702         drw_pmu->pmu = (struct pmu) {
 703                 .module         = THIS_MODULE,
 704                 .parent         = &pdev->dev,
 705                 .task_ctx_nr    = perf_invalid_context,
 706                 .event_init     = ali_drw_pmu_event_init,
 707                 .add            = ali_drw_pmu_add,
 708                 .del            = ali_drw_pmu_del,
 709                 .start          = ali_drw_pmu_start,
 710                 .stop           = ali_drw_pmu_stop,
 711                 .read           = ali_drw_pmu_read,
 712                 .attr_groups    = ali_drw_pmu_attr_groups,
 713                 .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
 714         };
 715
 716         ret = perf_pmu_register(&drw_pmu->pmu, name, -1);
 717         if (ret) {
 718                 dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n");
 719                 ali_drw_pmu_uninit_irq(drw_pmu);
 720         }
 721
 722         return ret;
 723 }
 724
 725 static void ali_drw_pmu_remove(struct platform_device *pdev)
 726 {
 727         struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev);
 728
 729         /* disable the generation of interrupt by all common counters */
 730         writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
 731                drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL);
 732
 733         ali_drw_pmu_uninit_irq(drw_pmu);
 734         perf_pmu_unregister(&drw_pmu->pmu);
 735 }
 736
 737 static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 738 {
 739         struct ali_drw_pmu_irq *irq;
 740         struct ali_drw_pmu *drw_pmu;
 741         unsigned int target;
 742
 743         irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node);
 744         if (cpu != irq->cpu)
 745                 return 0;
 746
 747         target = cpumask_any_and_but(cpumask_of_node(cpu_to_node(cpu)),
 748                                      cpu_online_mask, cpu);
 749         if (target >= nr_cpu_ids)
 750                 target = cpumask_any_but(cpu_online_mask, cpu);
 751
 752         if (target >= nr_cpu_ids)
 753                 return 0;
 754
 755         /* We're only reading, but this isn't the place to be involving RCU */
 756         mutex_lock(&ali_drw_pmu_irqs_lock);
 757         list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node)
 758                 perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target);
 759         mutex_unlock(&ali_drw_pmu_irqs_lock);
 760
 761         WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target)));
 762         irq->cpu = target;
 763
 764         return 0;
 765 }
 766
 767 /*
 768  * Due to historical reasons, the HID used in the production environment is
 769  * ARMHD700, so we leave ARMHD700 as Compatible ID.
 770  */
 771 static const struct acpi_device_id ali_drw_acpi_match[] = {
 772         {"BABA5000", 0},
 773         {"ARMHD700", 0},
 774         {}
 775 };
 776
 777 MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match);
 778
 779 static struct platform_driver ali_drw_pmu_driver = {
 780         .driver = {
 781                    .name = "ali_drw_pmu",
 782                    .acpi_match_table = ali_drw_acpi_match,
 783                    },
 784         .probe = ali_drw_pmu_probe,
 785         .remove_new = ali_drw_pmu_remove,
 786 };
 787
 788 static int __init ali_drw_pmu_init(void)
 789 {
 790         int ret;
 791
 792         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 793                                       "ali_drw_pmu:online",
 794                                       NULL, ali_drw_pmu_offline_cpu);
 795
 796         if (ret < 0) {
 797                 pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n",
 798                        ret);
 799                 return ret;
 800         }
 801         ali_drw_cpuhp_state_num = ret;
 802
 803         ret = platform_driver_register(&ali_drw_pmu_driver);
 804         if (ret)
 805                 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
 806
 807         return ret;
 808 }
 809
 810 static void __exit ali_drw_pmu_exit(void)
 811 {
 812         platform_driver_unregister(&ali_drw_pmu_driver);
 813         cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
 814 }
 815
 816 module_init(ali_drw_pmu_init);
 817 module_exit(ali_drw_pmu_exit);
 818
 819 MODULE_AUTHOR("Hongbo Yao <[email protected]>");
 820 MODULE_AUTHOR("Neng Chen <[email protected]>");
 821 MODULE_AUTHOR("Shuai Xue <[email protected]>");
 822 MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver");
 823 MODULE_LICENSE("GPL v2");