drivers/perf/riscv_pmu_sbi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * RISC-V performance counter support.
   4  *
   5  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
   6  *
   7  * This code is based on ARM perf event code which is in turn based on
   8  * sparc64 and x86 code.
   9  */
  10
  11 #define pr_fmt(fmt) "riscv-pmu-sbi: " fmt
  12
  13 #include <linux/mod_devicetable.h>
  14 #include <linux/perf/riscv_pmu.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/irq.h>
  17 #include <linux/irqdomain.h>
  18 #include <linux/of_irq.h>
  19 #include <linux/of.h>
  20 #include <linux/cpu_pm.h>
  21 #include <linux/sched/clock.h>
  22 #include <linux/soc/andes/irq.h>
  23 #include <linux/workqueue.h>
  24
  25 #include <asm/errata_list.h>
  26 #include <asm/sbi.h>
  27 #include <asm/cpufeature.h>
  28 #include <asm/vendor_extensions.h>
  29 #include <asm/vendor_extensions/andes.h>
  30
  31 #define ALT_SBI_PMU_OVERFLOW(__ovl)                                     \
  32 asm volatile(ALTERNATIVE_2(                                             \
  33         "csrr %0, " __stringify(CSR_SCOUNTOVF),                         \
  34         "csrr %0, " __stringify(THEAD_C9XX_CSR_SCOUNTEROF),             \
  35                 THEAD_VENDOR_ID, ERRATA_THEAD_PMU,                      \
  36                 CONFIG_ERRATA_THEAD_PMU,                                \
  37         "csrr %0, " __stringify(ANDES_CSR_SCOUNTEROF),                  \
  38                 ANDES_VENDOR_ID,                                        \
  39                 RISCV_ISA_VENDOR_EXT_XANDESPMU + RISCV_VENDOR_EXT_ALTERNATIVES_BASE, \
  40                 CONFIG_ANDES_CUSTOM_PMU)                                \
  41         : "=r" (__ovl) :                                                \
  42         : "memory")
  43
  44 #define ALT_SBI_PMU_OVF_CLEAR_PENDING(__irq_mask)                       \
  45 asm volatile(ALTERNATIVE(                                               \
  46         "csrc " __stringify(CSR_IP) ", %0\n\t",                         \
  47         "csrc " __stringify(ANDES_CSR_SLIP) ", %0\n\t",                 \
  48                 ANDES_VENDOR_ID,                                        \
  49                 RISCV_ISA_VENDOR_EXT_XANDESPMU + RISCV_VENDOR_EXT_ALTERNATIVES_BASE, \
  50                 CONFIG_ANDES_CUSTOM_PMU)                                \
  51         : : "r"(__irq_mask)                                             \
  52         : "memory")
  53
  54 #define SYSCTL_NO_USER_ACCESS   0
  55 #define SYSCTL_USER_ACCESS      1
  56 #define SYSCTL_LEGACY           2
  57
  58 #define PERF_EVENT_FLAG_NO_USER_ACCESS  BIT(SYSCTL_NO_USER_ACCESS)
  59 #define PERF_EVENT_FLAG_USER_ACCESS     BIT(SYSCTL_USER_ACCESS)
  60 #define PERF_EVENT_FLAG_LEGACY          BIT(SYSCTL_LEGACY)
  61
  62 PMU_FORMAT_ATTR(event, "config:0-47");
  63 PMU_FORMAT_ATTR(firmware, "config:62-63");
  64
  65 static bool sbi_v2_available;
  66 static DEFINE_STATIC_KEY_FALSE(sbi_pmu_snapshot_available);
  67 #define sbi_pmu_snapshot_available() \
  68         static_branch_unlikely(&sbi_pmu_snapshot_available)
  69
  70 static struct attribute *riscv_arch_formats_attr[] = {
  71         &format_attr_event.attr,
  72         &format_attr_firmware.attr,
  73         NULL,
  74 };
  75
  76 static struct attribute_group riscv_pmu_format_group = {
  77         .name = "format",
  78         .attrs = riscv_arch_formats_attr,
  79 };
  80
  81 static const struct attribute_group *riscv_pmu_attr_groups[] = {
  82         &riscv_pmu_format_group,
  83         NULL,
  84 };
  85
  86 /* Allow user mode access by default */
  87 static int sysctl_perf_user_access __read_mostly = SYSCTL_USER_ACCESS;
  88
  89 /*
  90  * RISC-V doesn't have heterogeneous harts yet. This need to be part of
  91  * per_cpu in case of harts with different pmu counters
  92  */
  93 static union sbi_pmu_ctr_info *pmu_ctr_list;
  94 static bool riscv_pmu_use_irq;
  95 static unsigned int riscv_pmu_irq_num;
  96 static unsigned int riscv_pmu_irq_mask;
  97 static unsigned int riscv_pmu_irq;
  98
  99 /* Cache the available counters in a bitmask */
 100 static unsigned long cmask;
 101
 102 struct sbi_pmu_event_data {
 103         union {
 104                 union {
 105                         struct hw_gen_event {
 106                                 uint32_t event_code:16;
 107                                 uint32_t event_type:4;
 108                                 uint32_t reserved:12;
 109                         } hw_gen_event;
 110                         struct hw_cache_event {
 111                                 uint32_t result_id:1;
 112                                 uint32_t op_id:2;
 113                                 uint32_t cache_id:13;
 114                                 uint32_t event_type:4;
 115                                 uint32_t reserved:12;
 116                         } hw_cache_event;
 117                 };
 118                 uint32_t event_idx;
 119         };
 120 };
 121
 122 static struct sbi_pmu_event_data pmu_hw_event_map[] = {
 123         [PERF_COUNT_HW_CPU_CYCLES]              = {.hw_gen_event = {
 124                                                         SBI_PMU_HW_CPU_CYCLES,
 125                                                         SBI_PMU_EVENT_TYPE_HW, 0}},
 126         [PERF_COUNT_HW_INSTRUCTIONS]            = {.hw_gen_event = {
 127                                                         SBI_PMU_HW_INSTRUCTIONS,
 128                                                         SBI_PMU_EVENT_TYPE_HW, 0}},
 129         [PERF_COUNT_HW_CACHE_REFERENCES]        = {.hw_gen_event = {
 130                                                         SBI_PMU_HW_CACHE_REFERENCES,
 131                                                         SBI_PMU_EVENT_TYPE_HW, 0}},
 132         [PERF_COUNT_HW_CACHE_MISSES]            = {.hw_gen_event = {
 133                                                         SBI_PMU_HW_CACHE_MISSES,
 134                                                         SBI_PMU_EVENT_TYPE_HW, 0}},
 135         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = {.hw_gen_event = {
 136                                                         SBI_PMU_HW_BRANCH_INSTRUCTIONS,
 137                                                         SBI_PMU_EVENT_TYPE_HW, 0}},
 138         [PERF_COUNT_HW_BRANCH_MISSES]           = {.hw_gen_event = {
 139                                                         SBI_PMU_HW_BRANCH_MISSES,
 140                                                         SBI_PMU_EVENT_TYPE_HW, 0}},
 141         [PERF_COUNT_HW_BUS_CYCLES]              = {.hw_gen_event = {
 142                                                         SBI_PMU_HW_BUS_CYCLES,
 143                                                         SBI_PMU_EVENT_TYPE_HW, 0}},
 144         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = {.hw_gen_event = {
 145                                                         SBI_PMU_HW_STALLED_CYCLES_FRONTEND,
 146                                                         SBI_PMU_EVENT_TYPE_HW, 0}},
 147         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = {.hw_gen_event = {
 148                                                         SBI_PMU_HW_STALLED_CYCLES_BACKEND,
 149                                                         SBI_PMU_EVENT_TYPE_HW, 0}},
 150         [PERF_COUNT_HW_REF_CPU_CYCLES]          = {.hw_gen_event = {
 151                                                         SBI_PMU_HW_REF_CPU_CYCLES,
 152                                                         SBI_PMU_EVENT_TYPE_HW, 0}},
 153 };
 154
 155 #define C(x) PERF_COUNT_HW_CACHE_##x
 156 static struct sbi_pmu_event_data pmu_cache_event_map[PERF_COUNT_HW_CACHE_MAX]
 157 [PERF_COUNT_HW_CACHE_OP_MAX]
 158 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 159         [C(L1D)] = {
 160                 [C(OP_READ)] = {
 161                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 162                                         C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 163                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 164                                         C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 165                 },
 166                 [C(OP_WRITE)] = {
 167                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 168                                         C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 169                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 170                                         C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 171                 },
 172                 [C(OP_PREFETCH)] = {
 173                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 174                                         C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 175                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 176                                         C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 177                 },
 178         },
 179         [C(L1I)] = {
 180                 [C(OP_READ)] = {
 181                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 182                                         C(OP_READ), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 183                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), C(OP_READ),
 184                                         C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 185                 },
 186                 [C(OP_WRITE)] = {
 187                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 188                                         C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 189                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 190                                         C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 191                 },
 192                 [C(OP_PREFETCH)] = {
 193                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 194                                         C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 195                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 196                                         C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 197                 },
 198         },
 199         [C(LL)] = {
 200                 [C(OP_READ)] = {
 201                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 202                                         C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 203                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 204                                         C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 205                 },
 206                 [C(OP_WRITE)] = {
 207                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 208                                         C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 209                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 210                                         C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 211                 },
 212                 [C(OP_PREFETCH)] = {
 213                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 214                                         C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 215                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 216                                         C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 217                 },
 218         },
 219         [C(DTLB)] = {
 220                 [C(OP_READ)] = {
 221                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 222                                         C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 223                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 224                                         C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 225                 },
 226                 [C(OP_WRITE)] = {
 227                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 228                                         C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 229                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 230                                         C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 231                 },
 232                 [C(OP_PREFETCH)] = {
 233                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 234                                         C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 235                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 236                                         C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 237                 },
 238         },
 239         [C(ITLB)] = {
 240                 [C(OP_READ)] = {
 241                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 242                                         C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 243                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 244                                         C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 245                 },
 246                 [C(OP_WRITE)] = {
 247                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 248                                         C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 249                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 250                                         C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 251                 },
 252                 [C(OP_PREFETCH)] = {
 253                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 254                                         C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 255                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 256                                         C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 257                 },
 258         },
 259         [C(BPU)] = {
 260                 [C(OP_READ)] = {
 261                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 262                                         C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 263                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 264                                         C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 265                 },
 266                 [C(OP_WRITE)] = {
 267                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 268                                         C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 269                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 270                                         C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 271                 },
 272                 [C(OP_PREFETCH)] = {
 273                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 274                                         C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 275                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 276                                         C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 277                 },
 278         },
 279         [C(NODE)] = {
 280                 [C(OP_READ)] = {
 281                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 282                                         C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 283                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 284                                         C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 285                 },
 286                 [C(OP_WRITE)] = {
 287                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 288                                         C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 289                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 290                                         C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 291                 },
 292                 [C(OP_PREFETCH)] = {
 293                         [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
 294                                         C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 295                         [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
 296                                         C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
 297                 },
 298         },
 299 };
 300
 301 static void pmu_sbi_check_event(struct sbi_pmu_event_data *edata)
 302 {
 303         struct sbiret ret;
 304
 305         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH,
 306                         0, cmask, 0, edata->event_idx, 0, 0);
 307         if (!ret.error) {
 308                 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP,
 309                           ret.value, 0x1, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0);
 310         } else if (ret.error == SBI_ERR_NOT_SUPPORTED) {
 311                 /* This event cannot be monitored by any counter */
 312                 edata->event_idx = -EINVAL;
 313         }
 314 }
 315
 316 static void pmu_sbi_check_std_events(struct work_struct *work)
 317 {
 318         for (int i = 0; i < ARRAY_SIZE(pmu_hw_event_map); i++)
 319                 pmu_sbi_check_event(&pmu_hw_event_map[i]);
 320
 321         for (int i = 0; i < ARRAY_SIZE(pmu_cache_event_map); i++)
 322                 for (int j = 0; j < ARRAY_SIZE(pmu_cache_event_map[i]); j++)
 323                         for (int k = 0; k < ARRAY_SIZE(pmu_cache_event_map[i][j]); k++)
 324                                 pmu_sbi_check_event(&pmu_cache_event_map[i][j][k]);
 325 }
 326
 327 static DECLARE_WORK(check_std_events_work, pmu_sbi_check_std_events);
 328
 329 static int pmu_sbi_ctr_get_width(int idx)
 330 {
 331         return pmu_ctr_list[idx].width;
 332 }
 333
 334 static bool pmu_sbi_ctr_is_fw(int cidx)
 335 {
 336         union sbi_pmu_ctr_info *info;
 337
 338         info = &pmu_ctr_list[cidx];
 339         if (!info)
 340                 return false;
 341
 342         return (info->type == SBI_PMU_CTR_TYPE_FW) ? true : false;
 343 }
 344
 345 /*
 346  * Returns the counter width of a programmable counter and number of hardware
 347  * counters. As we don't support heterogeneous CPUs yet, it is okay to just
 348  * return the counter width of the first programmable counter.
 349  */
 350 int riscv_pmu_get_hpm_info(u32 *hw_ctr_width, u32 *num_hw_ctr)
 351 {
 352         int i;
 353         union sbi_pmu_ctr_info *info;
 354         u32 hpm_width = 0, hpm_count = 0;
 355
 356         if (!cmask)
 357                 return -EINVAL;
 358
 359         for_each_set_bit(i, &cmask, RISCV_MAX_COUNTERS) {
 360                 info = &pmu_ctr_list[i];
 361                 if (!info)
 362                         continue;
 363                 if (!hpm_width && info->csr != CSR_CYCLE && info->csr != CSR_INSTRET)
 364                         hpm_width = info->width;
 365                 if (info->type == SBI_PMU_CTR_TYPE_HW)
 366                         hpm_count++;
 367         }
 368
 369         *hw_ctr_width = hpm_width;
 370         *num_hw_ctr = hpm_count;
 371
 372         return 0;
 373 }
 374 EXPORT_SYMBOL_GPL(riscv_pmu_get_hpm_info);
 375
 376 static uint8_t pmu_sbi_csr_index(struct perf_event *event)
 377 {
 378         return pmu_ctr_list[event->hw.idx].csr - CSR_CYCLE;
 379 }
 380
 381 static unsigned long pmu_sbi_get_filter_flags(struct perf_event *event)
 382 {
 383         unsigned long cflags = 0;
 384         bool guest_events = false;
 385
 386         if (event->attr.config1 & RISCV_PMU_CONFIG1_GUEST_EVENTS)
 387                 guest_events = true;
 388         if (event->attr.exclude_kernel)
 389                 cflags |= guest_events ? SBI_PMU_CFG_FLAG_SET_VSINH : SBI_PMU_CFG_FLAG_SET_SINH;
 390         if (event->attr.exclude_user)
 391                 cflags |= guest_events ? SBI_PMU_CFG_FLAG_SET_VUINH : SBI_PMU_CFG_FLAG_SET_UINH;
 392         if (guest_events && event->attr.exclude_hv)
 393                 cflags |= SBI_PMU_CFG_FLAG_SET_SINH;
 394         if (event->attr.exclude_host)
 395                 cflags |= SBI_PMU_CFG_FLAG_SET_UINH | SBI_PMU_CFG_FLAG_SET_SINH;
 396         if (event->attr.exclude_guest)
 397                 cflags |= SBI_PMU_CFG_FLAG_SET_VSINH | SBI_PMU_CFG_FLAG_SET_VUINH;
 398
 399         return cflags;
 400 }
 401
 402 static int pmu_sbi_ctr_get_idx(struct perf_event *event)
 403 {
 404         struct hw_perf_event *hwc = &event->hw;
 405         struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
 406         struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
 407         struct sbiret ret;
 408         int idx;
 409         uint64_t cbase = 0, cmask = rvpmu->cmask;
 410         unsigned long cflags = 0;
 411
 412         cflags = pmu_sbi_get_filter_flags(event);
 413
 414         /*
 415          * In legacy mode, we have to force the fixed counters for those events
 416          * but not in the user access mode as we want to use the other counters
 417          * that support sampling/filtering.
 418          */
 419         if ((hwc->flags & PERF_EVENT_FLAG_LEGACY) && (event->attr.type == PERF_TYPE_HARDWARE)) {
 420                 if (event->attr.config == PERF_COUNT_HW_CPU_CYCLES) {
 421                         cflags |= SBI_PMU_CFG_FLAG_SKIP_MATCH;
 422                         cmask = 1;
 423                 } else if (event->attr.config == PERF_COUNT_HW_INSTRUCTIONS) {
 424                         cflags |= SBI_PMU_CFG_FLAG_SKIP_MATCH;
 425                         cmask = BIT(CSR_INSTRET - CSR_CYCLE);
 426                 }
 427         }
 428
 429         /* retrieve the available counter index */
 430 #if defined(CONFIG_32BIT)
 431         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase,
 432                         cmask, cflags, hwc->event_base, hwc->config,
 433                         hwc->config >> 32);
 434 #else
 435         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase,
 436                         cmask, cflags, hwc->event_base, hwc->config, 0);
 437 #endif
 438         if (ret.error) {
 439                 pr_debug("Not able to find a counter for event %lx config %llx\n",
 440                         hwc->event_base, hwc->config);
 441                 return sbi_err_map_linux_errno(ret.error);
 442         }
 443
 444         idx = ret.value;
 445         if (!test_bit(idx, &rvpmu->cmask) || !pmu_ctr_list[idx].value)
 446                 return -ENOENT;
 447
 448         /* Additional sanity check for the counter id */
 449         if (pmu_sbi_ctr_is_fw(idx)) {
 450                 if (!test_and_set_bit(idx, cpuc->used_fw_ctrs))
 451                         return idx;
 452         } else {
 453                 if (!test_and_set_bit(idx, cpuc->used_hw_ctrs))
 454                         return idx;
 455         }
 456
 457         return -ENOENT;
 458 }
 459
 460 static void pmu_sbi_ctr_clear_idx(struct perf_event *event)
 461 {
 462
 463         struct hw_perf_event *hwc = &event->hw;
 464         struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
 465         struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
 466         int idx = hwc->idx;
 467
 468         if (pmu_sbi_ctr_is_fw(idx))
 469                 clear_bit(idx, cpuc->used_fw_ctrs);
 470         else
 471                 clear_bit(idx, cpuc->used_hw_ctrs);
 472 }
 473
 474 static int pmu_event_find_cache(u64 config)
 475 {
 476         unsigned int cache_type, cache_op, cache_result, ret;
 477
 478         cache_type = (config >>  0) & 0xff;
 479         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 480                 return -EINVAL;
 481
 482         cache_op = (config >>  8) & 0xff;
 483         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 484                 return -EINVAL;
 485
 486         cache_result = (config >> 16) & 0xff;
 487         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 488                 return -EINVAL;
 489
 490         ret = pmu_cache_event_map[cache_type][cache_op][cache_result].event_idx;
 491
 492         return ret;
 493 }
 494
 495 static bool pmu_sbi_is_fw_event(struct perf_event *event)
 496 {
 497         u32 type = event->attr.type;
 498         u64 config = event->attr.config;
 499
 500         if ((type == PERF_TYPE_RAW) && ((config >> 63) == 1))
 501                 return true;
 502         else
 503                 return false;
 504 }
 505
 506 static int pmu_sbi_event_map(struct perf_event *event, u64 *econfig)
 507 {
 508         u32 type = event->attr.type;
 509         u64 config = event->attr.config;
 510         u64 raw_config_val;
 511         int ret;
 512
 513         /*
 514          * Ensure we are finished checking standard hardware events for
 515          * validity before allowing userspace to configure any events.
 516          */
 517         flush_work(&check_std_events_work);
 518
 519         switch (type) {
 520         case PERF_TYPE_HARDWARE:
 521                 if (config >= PERF_COUNT_HW_MAX)
 522                         return -EINVAL;
 523                 ret = pmu_hw_event_map[event->attr.config].event_idx;
 524                 break;
 525         case PERF_TYPE_HW_CACHE:
 526                 ret = pmu_event_find_cache(config);
 527                 break;
 528         case PERF_TYPE_RAW:
 529                 /*
 530                  * As per SBI specification, the upper 16 bits must be unused
 531                  * for a raw event.
 532                  * Bits 63:62 are used to distinguish between raw events
 533                  * 00 - Hardware raw event
 534                  * 10 - SBI firmware events
 535                  * 11 - Risc-V platform specific firmware event
 536                  */
 537                 raw_config_val = config & RISCV_PMU_RAW_EVENT_MASK;
 538                 switch (config >> 62) {
 539                 case 0:
 540                         ret = RISCV_PMU_RAW_EVENT_IDX;
 541                         *econfig = raw_config_val;
 542                         break;
 543                 case 2:
 544                         ret = (raw_config_val & 0xFFFF) |
 545                                 (SBI_PMU_EVENT_TYPE_FW << 16);
 546                         break;
 547                 case 3:
 548                         /*
 549                          * For Risc-V platform specific firmware events
 550                          * Event code - 0xFFFF
 551                          * Event data - raw event encoding
 552                          */
 553                         ret = SBI_PMU_EVENT_TYPE_FW << 16 | RISCV_PLAT_FW_EVENT;
 554                         *econfig = raw_config_val;
 555                         break;
 556                 }
 557                 break;
 558         default:
 559                 ret = -EINVAL;
 560                 break;
 561         }
 562
 563         return ret;
 564 }
 565
 566 static void pmu_sbi_snapshot_free(struct riscv_pmu *pmu)
 567 {
 568         int cpu;
 569
 570         for_each_possible_cpu(cpu) {
 571                 struct cpu_hw_events *cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu);
 572
 573                 if (!cpu_hw_evt->snapshot_addr)
 574                         continue;
 575
 576                 free_page((unsigned long)cpu_hw_evt->snapshot_addr);
 577                 cpu_hw_evt->snapshot_addr = NULL;
 578                 cpu_hw_evt->snapshot_addr_phys = 0;
 579         }
 580 }
 581
 582 static int pmu_sbi_snapshot_alloc(struct riscv_pmu *pmu)
 583 {
 584         int cpu;
 585         struct page *snapshot_page;
 586
 587         for_each_possible_cpu(cpu) {
 588                 struct cpu_hw_events *cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu);
 589
 590                 snapshot_page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
 591                 if (!snapshot_page) {
 592                         pmu_sbi_snapshot_free(pmu);
 593                         return -ENOMEM;
 594                 }
 595                 cpu_hw_evt->snapshot_addr = page_to_virt(snapshot_page);
 596                 cpu_hw_evt->snapshot_addr_phys = page_to_phys(snapshot_page);
 597         }
 598
 599         return 0;
 600 }
 601
 602 static int pmu_sbi_snapshot_disable(void)
 603 {
 604         struct sbiret ret;
 605
 606         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, SBI_SHMEM_DISABLE,
 607                         SBI_SHMEM_DISABLE, 0, 0, 0, 0);
 608         if (ret.error) {
 609                 pr_warn("failed to disable snapshot shared memory\n");
 610                 return sbi_err_map_linux_errno(ret.error);
 611         }
 612
 613         return 0;
 614 }
 615
 616 static int pmu_sbi_snapshot_setup(struct riscv_pmu *pmu, int cpu)
 617 {
 618         struct cpu_hw_events *cpu_hw_evt;
 619         struct sbiret ret = {0};
 620
 621         cpu_hw_evt = per_cpu_ptr(pmu->hw_events, cpu);
 622         if (!cpu_hw_evt->snapshot_addr_phys)
 623                 return -EINVAL;
 624
 625         if (cpu_hw_evt->snapshot_set_done)
 626                 return 0;
 627
 628         if (IS_ENABLED(CONFIG_32BIT))
 629                 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM,
 630                                 cpu_hw_evt->snapshot_addr_phys,
 631                                 (u64)(cpu_hw_evt->snapshot_addr_phys) >> 32, 0, 0, 0, 0);
 632         else
 633                 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM,
 634                                 cpu_hw_evt->snapshot_addr_phys, 0, 0, 0, 0, 0);
 635
 636         /* Free up the snapshot area memory and fall back to SBI PMU calls without snapshot */
 637         if (ret.error) {
 638                 if (ret.error != SBI_ERR_NOT_SUPPORTED)
 639                         pr_warn("pmu snapshot setup failed with error %ld\n", ret.error);
 640                 return sbi_err_map_linux_errno(ret.error);
 641         }
 642
 643         memset(cpu_hw_evt->snapshot_cval_shcopy, 0, sizeof(u64) * RISCV_MAX_COUNTERS);
 644         cpu_hw_evt->snapshot_set_done = true;
 645
 646         return 0;
 647 }
 648
 649 static u64 pmu_sbi_ctr_read(struct perf_event *event)
 650 {
 651         struct hw_perf_event *hwc = &event->hw;
 652         int idx = hwc->idx;
 653         struct sbiret ret;
 654         u64 val = 0;
 655         struct riscv_pmu *pmu = to_riscv_pmu(event->pmu);
 656         struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
 657         struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
 658         union sbi_pmu_ctr_info info = pmu_ctr_list[idx];
 659
 660         /* Read the value from the shared memory directly only if counter is stopped */
 661         if (sbi_pmu_snapshot_available() && (hwc->state & PERF_HES_STOPPED)) {
 662                 val = sdata->ctr_values[idx];
 663                 return val;
 664         }
 665
 666         if (pmu_sbi_is_fw_event(event)) {
 667                 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ,
 668                                 hwc->idx, 0, 0, 0, 0, 0);
 669                 if (ret.error)
 670                         return 0;
 671
 672                 val = ret.value;
 673                 if (IS_ENABLED(CONFIG_32BIT) && sbi_v2_available && info.width >= 32) {
 674                         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ_HI,
 675                                         hwc->idx, 0, 0, 0, 0, 0);
 676                         if (!ret.error)
 677                                 val |= ((u64)ret.value << 32);
 678                         else
 679                                 WARN_ONCE(1, "Unable to read upper 32 bits of firmware counter error: %ld\n",
 680                                           ret.error);
 681                 }
 682         } else {
 683                 val = riscv_pmu_ctr_read_csr(info.csr);
 684                 if (IS_ENABLED(CONFIG_32BIT))
 685                         val |= ((u64)riscv_pmu_ctr_read_csr(info.csr + 0x80)) << 32;
 686         }
 687
 688         return val;
 689 }
 690
 691 static void pmu_sbi_set_scounteren(void *arg)
 692 {
 693         struct perf_event *event = (struct perf_event *)arg;
 694
 695         if (event->hw.idx != -1)
 696                 csr_write(CSR_SCOUNTEREN,
 697                           csr_read(CSR_SCOUNTEREN) | BIT(pmu_sbi_csr_index(event)));
 698 }
 699
 700 static void pmu_sbi_reset_scounteren(void *arg)
 701 {
 702         struct perf_event *event = (struct perf_event *)arg;
 703
 704         if (event->hw.idx != -1)
 705                 csr_write(CSR_SCOUNTEREN,
 706                           csr_read(CSR_SCOUNTEREN) & ~BIT(pmu_sbi_csr_index(event)));
 707 }
 708
 709 static void pmu_sbi_ctr_start(struct perf_event *event, u64 ival)
 710 {
 711         struct sbiret ret;
 712         struct hw_perf_event *hwc = &event->hw;
 713         unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE;
 714
 715         /* There is no benefit setting SNAPSHOT FLAG for a single counter */
 716 #if defined(CONFIG_32BIT)
 717         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx,
 718                         1, flag, ival, ival >> 32, 0);
 719 #else
 720         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx,
 721                         1, flag, ival, 0, 0);
 722 #endif
 723         if (ret.error && (ret.error != SBI_ERR_ALREADY_STARTED))
 724                 pr_err("Starting counter idx %d failed with error %d\n",
 725                         hwc->idx, sbi_err_map_linux_errno(ret.error));
 726
 727         if ((hwc->flags & PERF_EVENT_FLAG_USER_ACCESS) &&
 728             (hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT))
 729                 pmu_sbi_set_scounteren((void *)event);
 730 }
 731
 732 static void pmu_sbi_ctr_stop(struct perf_event *event, unsigned long flag)
 733 {
 734         struct sbiret ret;
 735         struct hw_perf_event *hwc = &event->hw;
 736         struct riscv_pmu *pmu = to_riscv_pmu(event->pmu);
 737         struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
 738         struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
 739
 740         if ((hwc->flags & PERF_EVENT_FLAG_USER_ACCESS) &&
 741             (hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT))
 742                 pmu_sbi_reset_scounteren((void *)event);
 743
 744         if (sbi_pmu_snapshot_available())
 745                 flag |= SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT;
 746
 747         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, hwc->idx, 1, flag, 0, 0, 0);
 748         if (!ret.error && sbi_pmu_snapshot_available()) {
 749                 /*
 750                  * The counter snapshot is based on the index base specified by hwc->idx.
 751                  * The actual counter value is updated in shared memory at index 0 when counter
 752                  * mask is 0x01. To ensure accurate counter values, it's necessary to transfer
 753                  * the counter value to shared memory. However, if hwc->idx is zero, the counter
 754                  * value is already correctly updated in shared memory, requiring no further
 755                  * adjustment.
 756                  */
 757                 if (hwc->idx > 0) {
 758                         sdata->ctr_values[hwc->idx] = sdata->ctr_values[0];
 759                         sdata->ctr_values[0] = 0;
 760                 }
 761         } else if (ret.error && (ret.error != SBI_ERR_ALREADY_STOPPED) &&
 762                 flag != SBI_PMU_STOP_FLAG_RESET) {
 763                 pr_err("Stopping counter idx %d failed with error %d\n",
 764                         hwc->idx, sbi_err_map_linux_errno(ret.error));
 765         }
 766 }
 767
 768 static int pmu_sbi_find_num_ctrs(void)
 769 {
 770         struct sbiret ret;
 771
 772         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_NUM_COUNTERS, 0, 0, 0, 0, 0, 0);
 773         if (!ret.error)
 774                 return ret.value;
 775         else
 776                 return sbi_err_map_linux_errno(ret.error);
 777 }
 778
 779 static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask)
 780 {
 781         struct sbiret ret;
 782         int i, num_hw_ctr = 0, num_fw_ctr = 0;
 783         union sbi_pmu_ctr_info cinfo;
 784
 785         pmu_ctr_list = kcalloc(nctr, sizeof(*pmu_ctr_list), GFP_KERNEL);
 786         if (!pmu_ctr_list)
 787                 return -ENOMEM;
 788
 789         for (i = 0; i < nctr; i++) {
 790                 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_GET_INFO, i, 0, 0, 0, 0, 0);
 791                 if (ret.error)
 792                         /* The logical counter ids are not expected to be contiguous */
 793                         continue;
 794
 795                 *mask |= BIT(i);
 796
 797                 cinfo.value = ret.value;
 798                 if (cinfo.type == SBI_PMU_CTR_TYPE_FW)
 799                         num_fw_ctr++;
 800                 else
 801                         num_hw_ctr++;
 802                 pmu_ctr_list[i].value = cinfo.value;
 803         }
 804
 805         pr_info("%d firmware and %d hardware counters\n", num_fw_ctr, num_hw_ctr);
 806
 807         return 0;
 808 }
 809
 810 static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu)
 811 {
 812         /*
 813          * No need to check the error because we are disabling all the counters
 814          * which may include counters that are not enabled yet.
 815          */
 816         sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP,
 817                   0, pmu->cmask, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0);
 818 }
 819
 820 static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu)
 821 {
 822         struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
 823         struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
 824         unsigned long flag = 0;
 825         int i, idx;
 826         struct sbiret ret;
 827         u64 temp_ctr_overflow_mask = 0;
 828
 829         if (sbi_pmu_snapshot_available())
 830                 flag = SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT;
 831
 832         /* Reset the shadow copy to avoid save/restore any value from previous overflow */
 833         memset(cpu_hw_evt->snapshot_cval_shcopy, 0, sizeof(u64) * RISCV_MAX_COUNTERS);
 834
 835         for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
 836                 /* No need to check the error here as we can't do anything about the error */
 837                 ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, i * BITS_PER_LONG,
 838                                 cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0);
 839                 if (!ret.error && sbi_pmu_snapshot_available()) {
 840                         /* Save the counter values to avoid clobbering */
 841                         for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG)
 842                                 cpu_hw_evt->snapshot_cval_shcopy[i * BITS_PER_LONG + idx] =
 843                                                         sdata->ctr_values[idx];
 844                         /* Save the overflow mask to avoid clobbering */
 845                         temp_ctr_overflow_mask |= sdata->ctr_overflow_mask << (i * BITS_PER_LONG);
 846                 }
 847         }
 848
 849         /* Restore the counter values to the shared memory for used hw counters */
 850         if (sbi_pmu_snapshot_available()) {
 851                 for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS)
 852                         sdata->ctr_values[idx] = cpu_hw_evt->snapshot_cval_shcopy[idx];
 853                 if (temp_ctr_overflow_mask)
 854                         sdata->ctr_overflow_mask = temp_ctr_overflow_mask;
 855         }
 856 }
 857
 858 /*
 859  * This function starts all the used counters in two step approach.
 860  * Any counter that did not overflow can be start in a single step
 861  * while the overflowed counters need to be started with updated initialization
 862  * value.
 863  */
 864 static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt,
 865                                               u64 ctr_ovf_mask)
 866 {
 867         int idx = 0, i;
 868         struct perf_event *event;
 869         unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE;
 870         unsigned long ctr_start_mask = 0;
 871         uint64_t max_period;
 872         struct hw_perf_event *hwc;
 873         u64 init_val = 0;
 874
 875         for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
 876                 ctr_start_mask = cpu_hw_evt->used_hw_ctrs[i] & ~ctr_ovf_mask;
 877                 /* Start all the counters that did not overflow in a single shot */
 878                 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, i * BITS_PER_LONG, ctr_start_mask,
 879                         0, 0, 0, 0);
 880         }
 881
 882         /* Reinitialize and start all the counter that overflowed */
 883         while (ctr_ovf_mask) {
 884                 if (ctr_ovf_mask & 0x01) {
 885                         event = cpu_hw_evt->events[idx];
 886                         hwc = &event->hw;
 887                         max_period = riscv_pmu_ctr_get_width_mask(event);
 888                         init_val = local64_read(&hwc->prev_count) & max_period;
 889 #if defined(CONFIG_32BIT)
 890                         sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1,
 891                                   flag, init_val, init_val >> 32, 0);
 892 #else
 893                         sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1,
 894                                   flag, init_val, 0, 0);
 895 #endif
 896                         perf_event_update_userpage(event);
 897                 }
 898                 ctr_ovf_mask = ctr_ovf_mask >> 1;
 899                 idx++;
 900         }
 901 }
 902
 903 static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_evt,
 904                                                    u64 ctr_ovf_mask)
 905 {
 906         int i, idx = 0;
 907         struct perf_event *event;
 908         unsigned long flag = SBI_PMU_START_FLAG_INIT_SNAPSHOT;
 909         u64 max_period, init_val = 0;
 910         struct hw_perf_event *hwc;
 911         struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
 912
 913         for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) {
 914                 if (ctr_ovf_mask & BIT(idx)) {
 915                         event = cpu_hw_evt->events[idx];
 916                         hwc = &event->hw;
 917                         max_period = riscv_pmu_ctr_get_width_mask(event);
 918                         init_val = local64_read(&hwc->prev_count) & max_period;
 919                         cpu_hw_evt->snapshot_cval_shcopy[idx] = init_val;
 920                 }
 921                 /*
 922                  * We do not need to update the non-overflow counters the previous
 923                  * value should have been there already.
 924                  */
 925         }
 926
 927         for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) {
 928                 /* Restore the counter values to relative indices for used hw counters */
 929                 for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG)
 930                         sdata->ctr_values[idx] =
 931                                         cpu_hw_evt->snapshot_cval_shcopy[idx + i * BITS_PER_LONG];
 932                 /* Start all the counters in a single shot */
 933                 sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx * BITS_PER_LONG,
 934                           cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0);
 935         }
 936 }
 937
 938 static void pmu_sbi_start_overflow_mask(struct riscv_pmu *pmu,
 939                                         u64 ctr_ovf_mask)
 940 {
 941         struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
 942
 943         if (sbi_pmu_snapshot_available())
 944                 pmu_sbi_start_ovf_ctrs_snapshot(cpu_hw_evt, ctr_ovf_mask);
 945         else
 946                 pmu_sbi_start_ovf_ctrs_sbi(cpu_hw_evt, ctr_ovf_mask);
 947 }
 948
 949 static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev)
 950 {
 951         struct perf_sample_data data;
 952         struct pt_regs *regs;
 953         struct hw_perf_event *hw_evt;
 954         union sbi_pmu_ctr_info *info;
 955         int lidx, hidx, fidx;
 956         struct riscv_pmu *pmu;
 957         struct perf_event *event;
 958         u64 overflow;
 959         u64 overflowed_ctrs = 0;
 960         struct cpu_hw_events *cpu_hw_evt = dev;
 961         u64 start_clock = sched_clock();
 962         struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr;
 963
 964         if (WARN_ON_ONCE(!cpu_hw_evt))
 965                 return IRQ_NONE;
 966
 967         /* Firmware counter don't support overflow yet */
 968         fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS);
 969         if (fidx == RISCV_MAX_COUNTERS) {
 970                 csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num));
 971                 return IRQ_NONE;
 972         }
 973
 974         event = cpu_hw_evt->events[fidx];
 975         if (!event) {
 976                 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask);
 977                 return IRQ_NONE;
 978         }
 979
 980         pmu = to_riscv_pmu(event->pmu);
 981         pmu_sbi_stop_hw_ctrs(pmu);
 982
 983         /* Overflow status register should only be read after counter are stopped */
 984         if (sbi_pmu_snapshot_available())
 985                 overflow = sdata->ctr_overflow_mask;
 986         else
 987                 ALT_SBI_PMU_OVERFLOW(overflow);
 988
 989         /*
 990          * Overflow interrupt pending bit should only be cleared after stopping
 991          * all the counters to avoid any race condition.
 992          */
 993         ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask);
 994
 995         /* No overflow bit is set */
 996         if (!overflow)
 997                 return IRQ_NONE;
 998
 999         regs = get_irq_regs();
1000
1001         for_each_set_bit(lidx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) {
1002                 struct perf_event *event = cpu_hw_evt->events[lidx];
1003
1004                 /* Skip if invalid event or user did not request a sampling */
1005                 if (!event || !is_sampling_event(event))
1006                         continue;
1007
1008                 info = &pmu_ctr_list[lidx];
1009                 /* Do a sanity check */
1010                 if (!info || info->type != SBI_PMU_CTR_TYPE_HW)
1011                         continue;
1012
1013                 if (sbi_pmu_snapshot_available())
1014                         /* SBI implementation already updated the logical indicies */
1015                         hidx = lidx;
1016                 else
1017                         /* compute hardware counter index */
1018                         hidx = info->csr - CSR_CYCLE;
1019
1020                 /* check if the corresponding bit is set in sscountovf or overflow mask in shmem */
1021                 if (!(overflow & BIT(hidx)))
1022                         continue;
1023
1024                 /*
1025                  * Keep a track of overflowed counters so that they can be started
1026                  * with updated initial value.
1027                  */
1028                 overflowed_ctrs |= BIT(lidx);
1029                 hw_evt = &event->hw;
1030                 /* Update the event states here so that we know the state while reading */
1031                 hw_evt->state |= PERF_HES_STOPPED;
1032                 riscv_pmu_event_update(event);
1033                 hw_evt->state |= PERF_HES_UPTODATE;
1034                 perf_sample_data_init(&data, 0, hw_evt->last_period);
1035                 if (riscv_pmu_event_set_period(event)) {
1036                         /*
1037                          * Unlike other ISAs, RISC-V don't have to disable interrupts
1038                          * to avoid throttling here. As per the specification, the
1039                          * interrupt remains disabled until the OF bit is set.
1040                          * Interrupts are enabled again only during the start.
1041                          * TODO: We will need to stop the guest counters once
1042                          * virtualization support is added.
1043                          */
1044                         perf_event_overflow(event, &data, regs);
1045                 }
1046                 /* Reset the state as we are going to start the counter after the loop */
1047                 hw_evt->state = 0;
1048         }
1049
1050         pmu_sbi_start_overflow_mask(pmu, overflowed_ctrs);
1051         perf_sample_event_took(sched_clock() - start_clock);
1052
1053         return IRQ_HANDLED;
1054 }
1055
1056 static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node)
1057 {
1058         struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node);
1059         struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
1060
1061         /*
1062          * We keep enabling userspace access to CYCLE, TIME and INSTRET via the
1063          * legacy option but that will be removed in the future.
1064          */
1065         if (sysctl_perf_user_access == SYSCTL_LEGACY)
1066                 csr_write(CSR_SCOUNTEREN, 0x7);
1067         else
1068                 csr_write(CSR_SCOUNTEREN, 0x2);
1069
1070         /* Stop all the counters so that they can be enabled from perf */
1071         pmu_sbi_stop_all(pmu);
1072
1073         if (riscv_pmu_use_irq) {
1074                 cpu_hw_evt->irq = riscv_pmu_irq;
1075                 ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask);
1076                 enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE);
1077         }
1078
1079         if (sbi_pmu_snapshot_available())
1080                 return pmu_sbi_snapshot_setup(pmu, cpu);
1081
1082         return 0;
1083 }
1084
1085 static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node)
1086 {
1087         if (riscv_pmu_use_irq) {
1088                 disable_percpu_irq(riscv_pmu_irq);
1089         }
1090
1091         /* Disable all counters access for user mode now */
1092         csr_write(CSR_SCOUNTEREN, 0x0);
1093
1094         if (sbi_pmu_snapshot_available())
1095                 return pmu_sbi_snapshot_disable();
1096
1097         return 0;
1098 }
1099
1100 static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pdev)
1101 {
1102         int ret;
1103         struct cpu_hw_events __percpu *hw_events = pmu->hw_events;
1104         struct irq_domain *domain = NULL;
1105
1106         if (riscv_isa_extension_available(NULL, SSCOFPMF)) {
1107                 riscv_pmu_irq_num = RV_IRQ_PMU;
1108                 riscv_pmu_use_irq = true;
1109         } else if (IS_ENABLED(CONFIG_ERRATA_THEAD_PMU) &&
1110                    riscv_cached_mvendorid(0) == THEAD_VENDOR_ID &&
1111                    riscv_cached_marchid(0) == 0 &&
1112                    riscv_cached_mimpid(0) == 0) {
1113                 riscv_pmu_irq_num = THEAD_C9XX_RV_IRQ_PMU;
1114                 riscv_pmu_use_irq = true;
1115         } else if (riscv_has_vendor_extension_unlikely(ANDES_VENDOR_ID,
1116                                                        RISCV_ISA_VENDOR_EXT_XANDESPMU) &&
1117                    IS_ENABLED(CONFIG_ANDES_CUSTOM_PMU)) {
1118                 riscv_pmu_irq_num = ANDES_SLI_CAUSE_BASE + ANDES_RV_IRQ_PMOVI;
1119                 riscv_pmu_use_irq = true;
1120         }
1121
1122         riscv_pmu_irq_mask = BIT(riscv_pmu_irq_num % BITS_PER_LONG);
1123
1124         if (!riscv_pmu_use_irq)
1125                 return -EOPNOTSUPP;
1126
1127         domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(),
1128                                           DOMAIN_BUS_ANY);
1129         if (!domain) {
1130                 pr_err("Failed to find INTC IRQ root domain\n");
1131                 return -ENODEV;
1132         }
1133
1134         riscv_pmu_irq = irq_create_mapping(domain, riscv_pmu_irq_num);
1135         if (!riscv_pmu_irq) {
1136                 pr_err("Failed to map PMU interrupt for node\n");
1137                 return -ENODEV;
1138         }
1139
1140         ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events);
1141         if (ret) {
1142                 pr_err("registering percpu irq failed [%d]\n", ret);
1143                 return ret;
1144         }
1145
1146         return 0;
1147 }
1148
1149 #ifdef CONFIG_CPU_PM
1150 static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd,
1151                                 void *v)
1152 {
1153         struct riscv_pmu *rvpmu = container_of(b, struct riscv_pmu, riscv_pm_nb);
1154         struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
1155         int enabled = bitmap_weight(cpuc->used_hw_ctrs, RISCV_MAX_COUNTERS);
1156         struct perf_event *event;
1157         int idx;
1158
1159         if (!enabled)
1160                 return NOTIFY_OK;
1161
1162         for (idx = 0; idx < RISCV_MAX_COUNTERS; idx++) {
1163                 event = cpuc->events[idx];
1164                 if (!event)
1165                         continue;
1166
1167                 switch (cmd) {
1168                 case CPU_PM_ENTER:
1169                         /*
1170                          * Stop and update the counter
1171                          */
1172                         riscv_pmu_stop(event, PERF_EF_UPDATE);
1173                         break;
1174                 case CPU_PM_EXIT:
1175                 case CPU_PM_ENTER_FAILED:
1176                         /*
1177                          * Restore and enable the counter.
1178                          */
1179                         riscv_pmu_start(event, PERF_EF_RELOAD);
1180                         break;
1181                 default:
1182                         break;
1183                 }
1184         }
1185
1186         return NOTIFY_OK;
1187 }
1188
1189 static int riscv_pm_pmu_register(struct riscv_pmu *pmu)
1190 {
1191         pmu->riscv_pm_nb.notifier_call = riscv_pm_pmu_notify;
1192         return cpu_pm_register_notifier(&pmu->riscv_pm_nb);
1193 }
1194
1195 static void riscv_pm_pmu_unregister(struct riscv_pmu *pmu)
1196 {
1197         cpu_pm_unregister_notifier(&pmu->riscv_pm_nb);
1198 }
1199 #else
1200 static inline int riscv_pm_pmu_register(struct riscv_pmu *pmu) { return 0; }
1201 static inline void riscv_pm_pmu_unregister(struct riscv_pmu *pmu) { }
1202 #endif
1203
1204 static void riscv_pmu_destroy(struct riscv_pmu *pmu)
1205 {
1206         if (sbi_v2_available) {
1207                 if (sbi_pmu_snapshot_available()) {
1208                         pmu_sbi_snapshot_disable();
1209                         pmu_sbi_snapshot_free(pmu);
1210                 }
1211         }
1212         riscv_pm_pmu_unregister(pmu);
1213         cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node);
1214 }
1215
1216 static void pmu_sbi_event_init(struct perf_event *event)
1217 {
1218         /*
1219          * The permissions are set at event_init so that we do not depend
1220          * on the sysctl value that can change.
1221          */
1222         if (sysctl_perf_user_access == SYSCTL_NO_USER_ACCESS)
1223                 event->hw.flags |= PERF_EVENT_FLAG_NO_USER_ACCESS;
1224         else if (sysctl_perf_user_access == SYSCTL_USER_ACCESS)
1225                 event->hw.flags |= PERF_EVENT_FLAG_USER_ACCESS;
1226         else
1227                 event->hw.flags |= PERF_EVENT_FLAG_LEGACY;
1228 }
1229
1230 static void pmu_sbi_event_mapped(struct perf_event *event, struct mm_struct *mm)
1231 {
1232         if (event->hw.flags & PERF_EVENT_FLAG_NO_USER_ACCESS)
1233                 return;
1234
1235         if (event->hw.flags & PERF_EVENT_FLAG_LEGACY) {
1236                 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES &&
1237                     event->attr.config != PERF_COUNT_HW_INSTRUCTIONS) {
1238                         return;
1239                 }
1240         }
1241
1242         /*
1243          * The user mmapped the event to directly access it: this is where
1244          * we determine based on sysctl_perf_user_access if we grant userspace
1245          * the direct access to this event. That means that within the same
1246          * task, some events may be directly accessible and some other may not,
1247          * if the user changes the value of sysctl_perf_user_accesss in the
1248          * meantime.
1249          */
1250
1251         event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT;
1252
1253         /*
1254          * We must enable userspace access *before* advertising in the user page
1255          * that it is possible to do so to avoid any race.
1256          * And we must notify all cpus here because threads that currently run
1257          * on other cpus will try to directly access the counter too without
1258          * calling pmu_sbi_ctr_start.
1259          */
1260         if (event->hw.flags & PERF_EVENT_FLAG_USER_ACCESS)
1261                 on_each_cpu_mask(mm_cpumask(mm),
1262                                  pmu_sbi_set_scounteren, (void *)event, 1);
1263 }
1264
1265 static void pmu_sbi_event_unmapped(struct perf_event *event, struct mm_struct *mm)
1266 {
1267         if (event->hw.flags & PERF_EVENT_FLAG_NO_USER_ACCESS)
1268                 return;
1269
1270         if (event->hw.flags & PERF_EVENT_FLAG_LEGACY) {
1271                 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES &&
1272                     event->attr.config != PERF_COUNT_HW_INSTRUCTIONS) {
1273                         return;
1274                 }
1275         }
1276
1277         /*
1278          * Here we can directly remove user access since the user does not have
1279          * access to the user page anymore so we avoid the racy window where the
1280          * user could have read cap_user_rdpmc to true right before we disable
1281          * it.
1282          */
1283         event->hw.flags &= ~PERF_EVENT_FLAG_USER_READ_CNT;
1284
1285         if (event->hw.flags & PERF_EVENT_FLAG_USER_ACCESS)
1286                 on_each_cpu_mask(mm_cpumask(mm),
1287                                  pmu_sbi_reset_scounteren, (void *)event, 1);
1288 }
1289
1290 static void riscv_pmu_update_counter_access(void *info)
1291 {
1292         if (sysctl_perf_user_access == SYSCTL_LEGACY)
1293                 csr_write(CSR_SCOUNTEREN, 0x7);
1294         else
1295                 csr_write(CSR_SCOUNTEREN, 0x2);
1296 }
1297
1298 static int riscv_pmu_proc_user_access_handler(const struct ctl_table *table,
1299                                               int write, void *buffer,
1300                                               size_t *lenp, loff_t *ppos)
1301 {
1302         int prev = sysctl_perf_user_access;
1303         int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
1304
1305         /*
1306          * Test against the previous value since we clear SCOUNTEREN when
1307          * sysctl_perf_user_access is set to SYSCTL_USER_ACCESS, but we should
1308          * not do that if that was already the case.
1309          */
1310         if (ret || !write || prev == sysctl_perf_user_access)
1311                 return ret;
1312
1313         on_each_cpu(riscv_pmu_update_counter_access, NULL, 1);
1314
1315         return 0;
1316 }
1317
1318 static struct ctl_table sbi_pmu_sysctl_table[] = {
1319         {
1320                 .procname       = "perf_user_access",
1321                 .data           = &sysctl_perf_user_access,
1322                 .maxlen         = sizeof(unsigned int),
1323                 .mode           = 0644,
1324                 .proc_handler   = riscv_pmu_proc_user_access_handler,
1325                 .extra1         = SYSCTL_ZERO,
1326                 .extra2         = SYSCTL_TWO,
1327         },
1328 };
1329
1330 static int pmu_sbi_device_probe(struct platform_device *pdev)
1331 {
1332         struct riscv_pmu *pmu = NULL;
1333         int ret = -ENODEV;
1334         int num_counters;
1335
1336         pr_info("SBI PMU extension is available\n");
1337         pmu = riscv_pmu_alloc();
1338         if (!pmu)
1339                 return -ENOMEM;
1340
1341         num_counters = pmu_sbi_find_num_ctrs();
1342         if (num_counters < 0) {
1343                 pr_err("SBI PMU extension doesn't provide any counters\n");
1344                 goto out_free;
1345         }
1346
1347         /* It is possible to get from SBI more than max number of counters */
1348         if (num_counters > RISCV_MAX_COUNTERS) {
1349                 num_counters = RISCV_MAX_COUNTERS;
1350                 pr_info("SBI returned more than maximum number of counters. Limiting the number of counters to %d\n", num_counters);
1351         }
1352
1353         /* cache all the information about counters now */
1354         if (pmu_sbi_get_ctrinfo(num_counters, &cmask))
1355                 goto out_free;
1356
1357         ret = pmu_sbi_setup_irqs(pmu, pdev);
1358         if (ret < 0) {
1359                 pr_info("Perf sampling/filtering is not supported as sscof extension is not available\n");
1360                 pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1361                 pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE;
1362         }
1363
1364         pmu->pmu.attr_groups = riscv_pmu_attr_groups;
1365         pmu->pmu.parent = &pdev->dev;
1366         pmu->cmask = cmask;
1367         pmu->ctr_start = pmu_sbi_ctr_start;
1368         pmu->ctr_stop = pmu_sbi_ctr_stop;
1369         pmu->event_map = pmu_sbi_event_map;
1370         pmu->ctr_get_idx = pmu_sbi_ctr_get_idx;
1371         pmu->ctr_get_width = pmu_sbi_ctr_get_width;
1372         pmu->ctr_clear_idx = pmu_sbi_ctr_clear_idx;
1373         pmu->ctr_read = pmu_sbi_ctr_read;
1374         pmu->event_init = pmu_sbi_event_init;
1375         pmu->event_mapped = pmu_sbi_event_mapped;
1376         pmu->event_unmapped = pmu_sbi_event_unmapped;
1377         pmu->csr_index = pmu_sbi_csr_index;
1378
1379         ret = riscv_pm_pmu_register(pmu);
1380         if (ret)
1381                 goto out_unregister;
1382
1383         ret = perf_pmu_register(&pmu->pmu, "cpu", PERF_TYPE_RAW);
1384         if (ret)
1385                 goto out_unregister;
1386
1387         /* SBI PMU Snapsphot is only available in SBI v2.0 */
1388         if (sbi_v2_available) {
1389                 int cpu;
1390
1391                 ret = pmu_sbi_snapshot_alloc(pmu);
1392                 if (ret)
1393                         goto out_unregister;
1394
1395                 cpu = get_cpu();
1396
1397                 ret = pmu_sbi_snapshot_setup(pmu, cpu);
1398                 if (ret) {
1399                         /* Snapshot is an optional feature. Continue if not available */
1400                         pmu_sbi_snapshot_free(pmu);
1401                 } else {
1402                         pr_info("SBI PMU snapshot detected\n");
1403                         /*
1404                          * We enable it once here for the boot cpu. If snapshot shmem setup
1405                          * fails during cpu hotplug process, it will fail to start the cpu
1406                          * as we can not handle hetergenous PMUs with different snapshot
1407                          * capability.
1408                          */
1409                         static_branch_enable(&sbi_pmu_snapshot_available);
1410                 }
1411                 put_cpu();
1412         }
1413
1414         register_sysctl("kernel", sbi_pmu_sysctl_table);
1415
1416         ret = cpuhp_state_add_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node);
1417         if (ret)
1418                 goto out_unregister;
1419
1420         /* Asynchronously check which standard events are available */
1421         schedule_work(&check_std_events_work);
1422
1423         return 0;
1424
1425 out_unregister:
1426         riscv_pmu_destroy(pmu);
1427
1428 out_free:
1429         kfree(pmu);
1430         return ret;
1431 }
1432
1433 static struct platform_driver pmu_sbi_driver = {
1434         .probe          = pmu_sbi_device_probe,
1435         .driver         = {
1436                 .name   = RISCV_PMU_SBI_PDEV_NAME,
1437         },
1438 };
1439
1440 static int __init pmu_sbi_devinit(void)
1441 {
1442         int ret;
1443         struct platform_device *pdev;
1444
1445         if (sbi_spec_version < sbi_mk_version(0, 3) ||
1446             !sbi_probe_extension(SBI_EXT_PMU)) {
1447                 return 0;
1448         }
1449
1450         if (sbi_spec_version >= sbi_mk_version(2, 0))
1451                 sbi_v2_available = true;
1452
1453         ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_RISCV_STARTING,
1454                                       "perf/riscv/pmu:starting",
1455                                       pmu_sbi_starting_cpu, pmu_sbi_dying_cpu);
1456         if (ret) {
1457                 pr_err("CPU hotplug notifier could not be registered: %d\n",
1458                        ret);
1459                 return ret;
1460         }
1461
1462         ret = platform_driver_register(&pmu_sbi_driver);
1463         if (ret)
1464                 return ret;
1465
1466         pdev = platform_device_register_simple(RISCV_PMU_SBI_PDEV_NAME, -1, NULL, 0);
1467         if (IS_ERR(pdev)) {
1468                 platform_driver_unregister(&pmu_sbi_driver);
1469                 return PTR_ERR(pdev);
1470         }
1471
1472         /* Notify legacy implementation that SBI pmu is available*/
1473         riscv_pmu_legacy_skip_init();
1474
1475         return ret;
1476 }
1477 device_initcall(pmu_sbi_devinit)