arch/x86/kernel/cpu/perf_event_intel_uncore.h

   1 #include <linux/module.h>
   2 #include <linux/slab.h>
   3 #include <linux/pci.h>
   4 #include <linux/perf_event.h>
   5 #include "perf_event.h"
   6
   7 #define UNCORE_PMU_NAME_LEN             32
   8 #define UNCORE_PMU_HRTIMER_INTERVAL     (60LL * NSEC_PER_SEC)
   9
  10 #define UNCORE_FIXED_EVENT              0xff
  11 #define UNCORE_PMC_IDX_MAX_GENERIC      8
  12 #define UNCORE_PMC_IDX_FIXED            UNCORE_PMC_IDX_MAX_GENERIC
  13 #define UNCORE_PMC_IDX_MAX              (UNCORE_PMC_IDX_FIXED + 1)
  14
  15 #define UNCORE_PCI_DEV_DATA(type, idx)  ((type << 8) | idx)
  16 #define UNCORE_PCI_DEV_TYPE(data)       ((data >> 8) & 0xff)
  17 #define UNCORE_PCI_DEV_IDX(data)        (data & 0xff)
  18 #define UNCORE_EXTRA_PCI_DEV            0xff
  19 #define UNCORE_EXTRA_PCI_DEV_MAX        2
  20
  21 /* support up to 8 sockets */
  22 #define UNCORE_SOCKET_MAX               8
  23
  24 #define UNCORE_EVENT_CONSTRAINT(c, n) EVENT_CONSTRAINT(c, n, 0xff)
  25
  26 /* SNB event control */
  27 #define SNB_UNC_CTL_EV_SEL_MASK                 0x000000ff
  28 #define SNB_UNC_CTL_UMASK_MASK                  0x0000ff00
  29 #define SNB_UNC_CTL_EDGE_DET                    (1 << 18)
  30 #define SNB_UNC_CTL_EN                          (1 << 22)
  31 #define SNB_UNC_CTL_INVERT                      (1 << 23)
  32 #define SNB_UNC_CTL_CMASK_MASK                  0x1f000000
  33 #define NHM_UNC_CTL_CMASK_MASK                  0xff000000
  34 #define NHM_UNC_FIXED_CTR_CTL_EN                (1 << 0)
  35
  36 #define SNB_UNC_RAW_EVENT_MASK                  (SNB_UNC_CTL_EV_SEL_MASK | \
  37                                                  SNB_UNC_CTL_UMASK_MASK | \
  38                                                  SNB_UNC_CTL_EDGE_DET | \
  39                                                  SNB_UNC_CTL_INVERT | \
  40                                                  SNB_UNC_CTL_CMASK_MASK)
  41
  42 #define NHM_UNC_RAW_EVENT_MASK                  (SNB_UNC_CTL_EV_SEL_MASK | \
  43                                                  SNB_UNC_CTL_UMASK_MASK | \
  44                                                  SNB_UNC_CTL_EDGE_DET | \
  45                                                  SNB_UNC_CTL_INVERT | \
  46                                                  NHM_UNC_CTL_CMASK_MASK)
  47
  48 /* SNB global control register */
  49 #define SNB_UNC_PERF_GLOBAL_CTL                 0x391
  50 #define SNB_UNC_FIXED_CTR_CTRL                  0x394
  51 #define SNB_UNC_FIXED_CTR                       0x395
  52
  53 /* SNB uncore global control */
  54 #define SNB_UNC_GLOBAL_CTL_CORE_ALL             ((1 << 4) - 1)
  55 #define SNB_UNC_GLOBAL_CTL_EN                   (1 << 29)
  56
  57 /* SNB Cbo register */
  58 #define SNB_UNC_CBO_0_PERFEVTSEL0               0x700
  59 #define SNB_UNC_CBO_0_PER_CTR0                  0x706
  60 #define SNB_UNC_CBO_MSR_OFFSET                  0x10
  61
  62 /* NHM global control register */
  63 #define NHM_UNC_PERF_GLOBAL_CTL                 0x391
  64 #define NHM_UNC_FIXED_CTR                       0x394
  65 #define NHM_UNC_FIXED_CTR_CTRL                  0x395
  66
  67 /* NHM uncore global control */
  68 #define NHM_UNC_GLOBAL_CTL_EN_PC_ALL            ((1ULL << 8) - 1)
  69 #define NHM_UNC_GLOBAL_CTL_EN_FC                (1ULL << 32)
  70
  71 /* NHM uncore register */
  72 #define NHM_UNC_PERFEVTSEL0                     0x3c0
  73 #define NHM_UNC_UNCORE_PMC0                     0x3b0
  74
  75 /* SNB-EP Box level control */
  76 #define SNBEP_PMON_BOX_CTL_RST_CTRL     (1 << 0)
  77 #define SNBEP_PMON_BOX_CTL_RST_CTRS     (1 << 1)
  78 #define SNBEP_PMON_BOX_CTL_FRZ          (1 << 8)
  79 #define SNBEP_PMON_BOX_CTL_FRZ_EN       (1 << 16)
  80 #define SNBEP_PMON_BOX_CTL_INT          (SNBEP_PMON_BOX_CTL_RST_CTRL | \
  81                                          SNBEP_PMON_BOX_CTL_RST_CTRS | \
  82                                          SNBEP_PMON_BOX_CTL_FRZ_EN)
  83 /* SNB-EP event control */
  84 #define SNBEP_PMON_CTL_EV_SEL_MASK      0x000000ff
  85 #define SNBEP_PMON_CTL_UMASK_MASK       0x0000ff00
  86 #define SNBEP_PMON_CTL_RST              (1 << 17)
  87 #define SNBEP_PMON_CTL_EDGE_DET         (1 << 18)
  88 #define SNBEP_PMON_CTL_EV_SEL_EXT       (1 << 21)
  89 #define SNBEP_PMON_CTL_EN               (1 << 22)
  90 #define SNBEP_PMON_CTL_INVERT           (1 << 23)
  91 #define SNBEP_PMON_CTL_TRESH_MASK       0xff000000
  92 #define SNBEP_PMON_RAW_EVENT_MASK       (SNBEP_PMON_CTL_EV_SEL_MASK | \
  93                                          SNBEP_PMON_CTL_UMASK_MASK | \
  94                                          SNBEP_PMON_CTL_EDGE_DET | \
  95                                          SNBEP_PMON_CTL_INVERT | \
  96                                          SNBEP_PMON_CTL_TRESH_MASK)
  97
  98 /* SNB-EP Ubox event control */
  99 #define SNBEP_U_MSR_PMON_CTL_TRESH_MASK         0x1f000000
 100 #define SNBEP_U_MSR_PMON_RAW_EVENT_MASK         \
 101                                 (SNBEP_PMON_CTL_EV_SEL_MASK | \
 102                                  SNBEP_PMON_CTL_UMASK_MASK | \
 103                                  SNBEP_PMON_CTL_EDGE_DET | \
 104                                  SNBEP_PMON_CTL_INVERT | \
 105                                  SNBEP_U_MSR_PMON_CTL_TRESH_MASK)
 106
 107 #define SNBEP_CBO_PMON_CTL_TID_EN               (1 << 19)
 108 #define SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK       (SNBEP_PMON_RAW_EVENT_MASK | \
 109                                                  SNBEP_CBO_PMON_CTL_TID_EN)
 110
 111 /* SNB-EP PCU event control */
 112 #define SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK     0x0000c000
 113 #define SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK       0x1f000000
 114 #define SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT       (1 << 30)
 115 #define SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET     (1 << 31)
 116 #define SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK       \
 117                                 (SNBEP_PMON_CTL_EV_SEL_MASK | \
 118                                  SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \
 119                                  SNBEP_PMON_CTL_EDGE_DET | \
 120                                  SNBEP_PMON_CTL_EV_SEL_EXT | \
 121                                  SNBEP_PMON_CTL_INVERT | \
 122                                  SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK | \
 123                                  SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \
 124                                  SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET)
 125
 126 #define SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK       \
 127                                 (SNBEP_PMON_RAW_EVENT_MASK | \
 128                                  SNBEP_PMON_CTL_EV_SEL_EXT)
 129
 130 /* SNB-EP pci control register */
 131 #define SNBEP_PCI_PMON_BOX_CTL                  0xf4
 132 #define SNBEP_PCI_PMON_CTL0                     0xd8
 133 /* SNB-EP pci counter register */
 134 #define SNBEP_PCI_PMON_CTR0                     0xa0
 135
 136 /* SNB-EP home agent register */
 137 #define SNBEP_HA_PCI_PMON_BOX_ADDRMATCH0        0x40
 138 #define SNBEP_HA_PCI_PMON_BOX_ADDRMATCH1        0x44
 139 #define SNBEP_HA_PCI_PMON_BOX_OPCODEMATCH       0x48
 140 /* SNB-EP memory controller register */
 141 #define SNBEP_MC_CHy_PCI_PMON_FIXED_CTL         0xf0
 142 #define SNBEP_MC_CHy_PCI_PMON_FIXED_CTR         0xd0
 143 /* SNB-EP QPI register */
 144 #define SNBEP_Q_Py_PCI_PMON_PKT_MATCH0          0x228
 145 #define SNBEP_Q_Py_PCI_PMON_PKT_MATCH1          0x22c
 146 #define SNBEP_Q_Py_PCI_PMON_PKT_MASK0           0x238
 147 #define SNBEP_Q_Py_PCI_PMON_PKT_MASK1           0x23c
 148
 149 /* SNB-EP Ubox register */
 150 #define SNBEP_U_MSR_PMON_CTR0                   0xc16
 151 #define SNBEP_U_MSR_PMON_CTL0                   0xc10
 152
 153 #define SNBEP_U_MSR_PMON_UCLK_FIXED_CTL         0xc08
 154 #define SNBEP_U_MSR_PMON_UCLK_FIXED_CTR         0xc09
 155
 156 /* SNB-EP Cbo register */
 157 #define SNBEP_C0_MSR_PMON_CTR0                  0xd16
 158 #define SNBEP_C0_MSR_PMON_CTL0                  0xd10
 159 #define SNBEP_C0_MSR_PMON_BOX_CTL               0xd04
 160 #define SNBEP_C0_MSR_PMON_BOX_FILTER            0xd14
 161 #define SNBEP_CBO_MSR_OFFSET                    0x20
 162
 163 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_TID       0x1f
 164 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_NID       0x3fc00
 165 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE     0x7c0000
 166 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC       0xff800000
 167
 168 #define SNBEP_CBO_EVENT_EXTRA_REG(e, m, i) {    \
 169         .event = (e),                           \
 170         .msr = SNBEP_C0_MSR_PMON_BOX_FILTER,    \
 171         .config_mask = (m),                     \
 172         .idx = (i)                              \
 173 }
 174
 175 /* SNB-EP PCU register */
 176 #define SNBEP_PCU_MSR_PMON_CTR0                 0xc36
 177 #define SNBEP_PCU_MSR_PMON_CTL0                 0xc30
 178 #define SNBEP_PCU_MSR_PMON_BOX_CTL              0xc24
 179 #define SNBEP_PCU_MSR_PMON_BOX_FILTER           0xc34
 180 #define SNBEP_PCU_MSR_PMON_BOX_FILTER_MASK      0xffffffff
 181 #define SNBEP_PCU_MSR_CORE_C3_CTR               0x3fc
 182 #define SNBEP_PCU_MSR_CORE_C6_CTR               0x3fd
 183
 184 /* IVT event control */
 185 #define IVT_PMON_BOX_CTL_INT            (SNBEP_PMON_BOX_CTL_RST_CTRL | \
 186                                          SNBEP_PMON_BOX_CTL_RST_CTRS)
 187 #define IVT_PMON_RAW_EVENT_MASK         (SNBEP_PMON_CTL_EV_SEL_MASK | \
 188                                          SNBEP_PMON_CTL_UMASK_MASK | \
 189                                          SNBEP_PMON_CTL_EDGE_DET | \
 190                                          SNBEP_PMON_CTL_TRESH_MASK)
 191 /* IVT Ubox */
 192 #define IVT_U_MSR_PMON_GLOBAL_CTL               0xc00
 193 #define IVT_U_PMON_GLOBAL_FRZ_ALL               (1 << 31)
 194 #define IVT_U_PMON_GLOBAL_UNFRZ_ALL             (1 << 29)
 195
 196 #define IVT_U_MSR_PMON_RAW_EVENT_MASK   \
 197                                 (SNBEP_PMON_CTL_EV_SEL_MASK | \
 198                                  SNBEP_PMON_CTL_UMASK_MASK | \
 199                                  SNBEP_PMON_CTL_EDGE_DET | \
 200                                  SNBEP_U_MSR_PMON_CTL_TRESH_MASK)
 201 /* IVT Cbo */
 202 #define IVT_CBO_MSR_PMON_RAW_EVENT_MASK         (IVT_PMON_RAW_EVENT_MASK | \
 203                                                  SNBEP_CBO_PMON_CTL_TID_EN)
 204
 205 #define IVT_CB0_MSR_PMON_BOX_FILTER_TID         (0x1fULL << 0)
 206 #define IVT_CB0_MSR_PMON_BOX_FILTER_LINK        (0xfULL << 5)
 207 #define IVT_CB0_MSR_PMON_BOX_FILTER_STATE       (0x3fULL << 17)
 208 #define IVT_CB0_MSR_PMON_BOX_FILTER_NID         (0xffffULL << 32)
 209 #define IVT_CB0_MSR_PMON_BOX_FILTER_OPC         (0x1ffULL << 52)
 210 #define IVT_CB0_MSR_PMON_BOX_FILTER_C6          (0x1ULL << 61)
 211 #define IVT_CB0_MSR_PMON_BOX_FILTER_NC          (0x1ULL << 62)
 212 #define IVT_CB0_MSR_PMON_BOX_FILTER_IOSC        (0x1ULL << 63)
 213
 214 /* IVT home agent */
 215 #define IVT_HA_PCI_PMON_CTL_Q_OCC_RST           (1 << 16)
 216 #define IVT_HA_PCI_PMON_RAW_EVENT_MASK          \
 217                                 (IVT_PMON_RAW_EVENT_MASK | \
 218                                  IVT_HA_PCI_PMON_CTL_Q_OCC_RST)
 219 /* IVT PCU */
 220 #define IVT_PCU_MSR_PMON_RAW_EVENT_MASK \
 221                                 (SNBEP_PMON_CTL_EV_SEL_MASK | \
 222                                  SNBEP_PMON_CTL_EV_SEL_EXT | \
 223                                  SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \
 224                                  SNBEP_PMON_CTL_EDGE_DET | \
 225                                  SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK | \
 226                                  SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \
 227                                  SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET)
 228 /* IVT QPI */
 229 #define IVT_QPI_PCI_PMON_RAW_EVENT_MASK \
 230                                 (IVT_PMON_RAW_EVENT_MASK | \
 231                                  SNBEP_PMON_CTL_EV_SEL_EXT)
 232
 233 /* NHM-EX event control */
 234 #define NHMEX_PMON_CTL_EV_SEL_MASK      0x000000ff
 235 #define NHMEX_PMON_CTL_UMASK_MASK       0x0000ff00
 236 #define NHMEX_PMON_CTL_EN_BIT0          (1 << 0)
 237 #define NHMEX_PMON_CTL_EDGE_DET         (1 << 18)
 238 #define NHMEX_PMON_CTL_PMI_EN           (1 << 20)
 239 #define NHMEX_PMON_CTL_EN_BIT22         (1 << 22)
 240 #define NHMEX_PMON_CTL_INVERT           (1 << 23)
 241 #define NHMEX_PMON_CTL_TRESH_MASK       0xff000000
 242 #define NHMEX_PMON_RAW_EVENT_MASK       (NHMEX_PMON_CTL_EV_SEL_MASK | \
 243                                          NHMEX_PMON_CTL_UMASK_MASK | \
 244                                          NHMEX_PMON_CTL_EDGE_DET | \
 245                                          NHMEX_PMON_CTL_INVERT | \
 246                                          NHMEX_PMON_CTL_TRESH_MASK)
 247
 248 /* NHM-EX Ubox */
 249 #define NHMEX_U_MSR_PMON_GLOBAL_CTL             0xc00
 250 #define NHMEX_U_MSR_PMON_CTR                    0xc11
 251 #define NHMEX_U_MSR_PMON_EV_SEL                 0xc10
 252
 253 #define NHMEX_U_PMON_GLOBAL_EN                  (1 << 0)
 254 #define NHMEX_U_PMON_GLOBAL_PMI_CORE_SEL        0x0000001e
 255 #define NHMEX_U_PMON_GLOBAL_EN_ALL              (1 << 28)
 256 #define NHMEX_U_PMON_GLOBAL_RST_ALL             (1 << 29)
 257 #define NHMEX_U_PMON_GLOBAL_FRZ_ALL             (1 << 31)
 258
 259 #define NHMEX_U_PMON_RAW_EVENT_MASK             \
 260                 (NHMEX_PMON_CTL_EV_SEL_MASK |   \
 261                  NHMEX_PMON_CTL_EDGE_DET)
 262
 263 /* NHM-EX Cbox */
 264 #define NHMEX_C0_MSR_PMON_GLOBAL_CTL            0xd00
 265 #define NHMEX_C0_MSR_PMON_CTR0                  0xd11
 266 #define NHMEX_C0_MSR_PMON_EV_SEL0               0xd10
 267 #define NHMEX_C_MSR_OFFSET                      0x20
 268
 269 /* NHM-EX Bbox */
 270 #define NHMEX_B0_MSR_PMON_GLOBAL_CTL            0xc20
 271 #define NHMEX_B0_MSR_PMON_CTR0                  0xc31
 272 #define NHMEX_B0_MSR_PMON_CTL0                  0xc30
 273 #define NHMEX_B_MSR_OFFSET                      0x40
 274 #define NHMEX_B0_MSR_MATCH                      0xe45
 275 #define NHMEX_B0_MSR_MASK                       0xe46
 276 #define NHMEX_B1_MSR_MATCH                      0xe4d
 277 #define NHMEX_B1_MSR_MASK                       0xe4e
 278
 279 #define NHMEX_B_PMON_CTL_EN                     (1 << 0)
 280 #define NHMEX_B_PMON_CTL_EV_SEL_SHIFT           1
 281 #define NHMEX_B_PMON_CTL_EV_SEL_MASK            \
 282                 (0x1f << NHMEX_B_PMON_CTL_EV_SEL_SHIFT)
 283 #define NHMEX_B_PMON_CTR_SHIFT          6
 284 #define NHMEX_B_PMON_CTR_MASK           \
 285                 (0x3 << NHMEX_B_PMON_CTR_SHIFT)
 286 #define NHMEX_B_PMON_RAW_EVENT_MASK             \
 287                 (NHMEX_B_PMON_CTL_EV_SEL_MASK | \
 288                  NHMEX_B_PMON_CTR_MASK)
 289
 290 /* NHM-EX Sbox */
 291 #define NHMEX_S0_MSR_PMON_GLOBAL_CTL            0xc40
 292 #define NHMEX_S0_MSR_PMON_CTR0                  0xc51
 293 #define NHMEX_S0_MSR_PMON_CTL0                  0xc50
 294 #define NHMEX_S_MSR_OFFSET                      0x80
 295 #define NHMEX_S0_MSR_MM_CFG                     0xe48
 296 #define NHMEX_S0_MSR_MATCH                      0xe49
 297 #define NHMEX_S0_MSR_MASK                       0xe4a
 298 #define NHMEX_S1_MSR_MM_CFG                     0xe58
 299 #define NHMEX_S1_MSR_MATCH                      0xe59
 300 #define NHMEX_S1_MSR_MASK                       0xe5a
 301
 302 #define NHMEX_S_PMON_MM_CFG_EN                  (0x1ULL << 63)
 303 #define NHMEX_S_EVENT_TO_R_PROG_EV              0
 304
 305 /* NHM-EX Mbox */
 306 #define NHMEX_M0_MSR_GLOBAL_CTL                 0xca0
 307 #define NHMEX_M0_MSR_PMU_DSP                    0xca5
 308 #define NHMEX_M0_MSR_PMU_ISS                    0xca6
 309 #define NHMEX_M0_MSR_PMU_MAP                    0xca7
 310 #define NHMEX_M0_MSR_PMU_MSC_THR                0xca8
 311 #define NHMEX_M0_MSR_PMU_PGT                    0xca9
 312 #define NHMEX_M0_MSR_PMU_PLD                    0xcaa
 313 #define NHMEX_M0_MSR_PMU_ZDP_CTL_FVC            0xcab
 314 #define NHMEX_M0_MSR_PMU_CTL0                   0xcb0
 315 #define NHMEX_M0_MSR_PMU_CNT0                   0xcb1
 316 #define NHMEX_M_MSR_OFFSET                      0x40
 317 #define NHMEX_M0_MSR_PMU_MM_CFG                 0xe54
 318 #define NHMEX_M1_MSR_PMU_MM_CFG                 0xe5c
 319
 320 #define NHMEX_M_PMON_MM_CFG_EN                  (1ULL << 63)
 321 #define NHMEX_M_PMON_ADDR_MATCH_MASK            0x3ffffffffULL
 322 #define NHMEX_M_PMON_ADDR_MASK_MASK             0x7ffffffULL
 323 #define NHMEX_M_PMON_ADDR_MASK_SHIFT            34
 324
 325 #define NHMEX_M_PMON_CTL_EN                     (1 << 0)
 326 #define NHMEX_M_PMON_CTL_PMI_EN                 (1 << 1)
 327 #define NHMEX_M_PMON_CTL_COUNT_MODE_SHIFT       2
 328 #define NHMEX_M_PMON_CTL_COUNT_MODE_MASK        \
 329         (0x3 << NHMEX_M_PMON_CTL_COUNT_MODE_SHIFT)
 330 #define NHMEX_M_PMON_CTL_STORAGE_MODE_SHIFT     4
 331 #define NHMEX_M_PMON_CTL_STORAGE_MODE_MASK      \
 332         (0x3 << NHMEX_M_PMON_CTL_STORAGE_MODE_SHIFT)
 333 #define NHMEX_M_PMON_CTL_WRAP_MODE              (1 << 6)
 334 #define NHMEX_M_PMON_CTL_FLAG_MODE              (1 << 7)
 335 #define NHMEX_M_PMON_CTL_INC_SEL_SHIFT          9
 336 #define NHMEX_M_PMON_CTL_INC_SEL_MASK           \
 337         (0x1f << NHMEX_M_PMON_CTL_INC_SEL_SHIFT)
 338 #define NHMEX_M_PMON_CTL_SET_FLAG_SEL_SHIFT     19
 339 #define NHMEX_M_PMON_CTL_SET_FLAG_SEL_MASK      \
 340         (0x7 << NHMEX_M_PMON_CTL_SET_FLAG_SEL_SHIFT)
 341 #define NHMEX_M_PMON_RAW_EVENT_MASK                     \
 342                 (NHMEX_M_PMON_CTL_COUNT_MODE_MASK |     \
 343                  NHMEX_M_PMON_CTL_STORAGE_MODE_MASK |   \
 344                  NHMEX_M_PMON_CTL_WRAP_MODE |           \
 345                  NHMEX_M_PMON_CTL_FLAG_MODE |           \
 346                  NHMEX_M_PMON_CTL_INC_SEL_MASK |        \
 347                  NHMEX_M_PMON_CTL_SET_FLAG_SEL_MASK)
 348
 349 #define NHMEX_M_PMON_ZDP_CTL_FVC_MASK           (((1 << 11) - 1) | (1 << 23))
 350 #define NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(n)  (0x7ULL << (11 + 3 * (n)))
 351
 352 #define WSMEX_M_PMON_ZDP_CTL_FVC_MASK           (((1 << 12) - 1) | (1 << 24))
 353 #define WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(n)  (0x7ULL << (12 + 3 * (n)))
 354
 355 /*
 356  * use the 9~13 bits to select event If the 7th bit is not set,
 357  * otherwise use the 19~21 bits to select event.
 358  */
 359 #define MBOX_INC_SEL(x) ((x) << NHMEX_M_PMON_CTL_INC_SEL_SHIFT)
 360 #define MBOX_SET_FLAG_SEL(x) (((x) << NHMEX_M_PMON_CTL_SET_FLAG_SEL_SHIFT) | \
 361                                 NHMEX_M_PMON_CTL_FLAG_MODE)
 362 #define MBOX_INC_SEL_MASK (NHMEX_M_PMON_CTL_INC_SEL_MASK | \
 363                            NHMEX_M_PMON_CTL_FLAG_MODE)
 364 #define MBOX_SET_FLAG_SEL_MASK (NHMEX_M_PMON_CTL_SET_FLAG_SEL_MASK | \
 365                                 NHMEX_M_PMON_CTL_FLAG_MODE)
 366 #define MBOX_INC_SEL_EXTAR_REG(c, r) \
 367                 EVENT_EXTRA_REG(MBOX_INC_SEL(c), NHMEX_M0_MSR_PMU_##r, \
 368                                 MBOX_INC_SEL_MASK, (u64)-1, NHMEX_M_##r)
 369 #define MBOX_SET_FLAG_SEL_EXTRA_REG(c, r) \
 370                 EVENT_EXTRA_REG(MBOX_SET_FLAG_SEL(c), NHMEX_M0_MSR_PMU_##r, \
 371                                 MBOX_SET_FLAG_SEL_MASK, \
 372                                 (u64)-1, NHMEX_M_##r)
 373
 374 /* NHM-EX Rbox */
 375 #define NHMEX_R_MSR_GLOBAL_CTL                  0xe00
 376 #define NHMEX_R_MSR_PMON_CTL0                   0xe10
 377 #define NHMEX_R_MSR_PMON_CNT0                   0xe11
 378 #define NHMEX_R_MSR_OFFSET                      0x20
 379
 380 #define NHMEX_R_MSR_PORTN_QLX_CFG(n)            \
 381                 ((n) < 4 ? (0xe0c + (n)) : (0xe2c + (n) - 4))
 382 #define NHMEX_R_MSR_PORTN_IPERF_CFG0(n)         (0xe04 + (n))
 383 #define NHMEX_R_MSR_PORTN_IPERF_CFG1(n)         (0xe24 + (n))
 384 #define NHMEX_R_MSR_PORTN_XBR_OFFSET(n)         \
 385                 (((n) < 4 ? 0 : 0x10) + (n) * 4)
 386 #define NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(n)    \
 387                 (0xe60 + NHMEX_R_MSR_PORTN_XBR_OFFSET(n))
 388 #define NHMEX_R_MSR_PORTN_XBR_SET1_MATCH(n)     \
 389                 (NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(n) + 1)
 390 #define NHMEX_R_MSR_PORTN_XBR_SET1_MASK(n)      \
 391                 (NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(n) + 2)
 392 #define NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(n)    \
 393                 (0xe70 + NHMEX_R_MSR_PORTN_XBR_OFFSET(n))
 394 #define NHMEX_R_MSR_PORTN_XBR_SET2_MATCH(n)     \
 395                 (NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(n) + 1)
 396 #define NHMEX_R_MSR_PORTN_XBR_SET2_MASK(n)      \
 397                 (NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(n) + 2)
 398
 399 #define NHMEX_R_PMON_CTL_EN                     (1 << 0)
 400 #define NHMEX_R_PMON_CTL_EV_SEL_SHIFT           1
 401 #define NHMEX_R_PMON_CTL_EV_SEL_MASK            \
 402                 (0x1f << NHMEX_R_PMON_CTL_EV_SEL_SHIFT)
 403 #define NHMEX_R_PMON_CTL_PMI_EN                 (1 << 6)
 404 #define NHMEX_R_PMON_RAW_EVENT_MASK             NHMEX_R_PMON_CTL_EV_SEL_MASK
 405
 406 /* NHM-EX Wbox */
 407 #define NHMEX_W_MSR_GLOBAL_CTL                  0xc80
 408 #define NHMEX_W_MSR_PMON_CNT0                   0xc90
 409 #define NHMEX_W_MSR_PMON_EVT_SEL0               0xc91
 410 #define NHMEX_W_MSR_PMON_FIXED_CTR              0x394
 411 #define NHMEX_W_MSR_PMON_FIXED_CTL              0x395
 412
 413 #define NHMEX_W_PMON_GLOBAL_FIXED_EN            (1ULL << 31)
 414
 415 struct intel_uncore_ops;
 416 struct intel_uncore_pmu;
 417 struct intel_uncore_box;
 418 struct uncore_event_desc;
 419
 420 struct intel_uncore_type {
 421         const char *name;
 422         int num_counters;
 423         int num_boxes;
 424         int perf_ctr_bits;
 425         int fixed_ctr_bits;
 426         unsigned perf_ctr;
 427         unsigned event_ctl;
 428         unsigned event_mask;
 429         unsigned fixed_ctr;
 430         unsigned fixed_ctl;
 431         unsigned box_ctl;
 432         unsigned msr_offset;
 433         unsigned num_shared_regs:8;
 434         unsigned single_fixed:1;
 435         unsigned pair_ctr_ctl:1;
 436         unsigned *msr_offsets;
 437         struct event_constraint unconstrainted;
 438         struct event_constraint *constraints;
 439         struct intel_uncore_pmu *pmus;
 440         struct intel_uncore_ops *ops;
 441         struct uncore_event_desc *event_descs;
 442         const struct attribute_group *attr_groups[4];
 443 };
 444
 445 #define pmu_group attr_groups[0]
 446 #define format_group attr_groups[1]
 447 #define events_group attr_groups[2]
 448
 449 struct intel_uncore_ops {
 450         void (*init_box)(struct intel_uncore_box *);
 451         void (*disable_box)(struct intel_uncore_box *);
 452         void (*enable_box)(struct intel_uncore_box *);
 453         void (*disable_event)(struct intel_uncore_box *, struct perf_event *);
 454         void (*enable_event)(struct intel_uncore_box *, struct perf_event *);
 455         u64 (*read_counter)(struct intel_uncore_box *, struct perf_event *);
 456         int (*hw_config)(struct intel_uncore_box *, struct perf_event *);
 457         struct event_constraint *(*get_constraint)(struct intel_uncore_box *,
 458                                                    struct perf_event *);
 459         void (*put_constraint)(struct intel_uncore_box *, struct perf_event *);
 460 };
 461
 462 struct intel_uncore_pmu {
 463         struct pmu pmu;
 464         char name[UNCORE_PMU_NAME_LEN];
 465         int pmu_idx;
 466         int func_id;
 467         struct intel_uncore_type *type;
 468         struct intel_uncore_box ** __percpu box;
 469         struct list_head box_list;
 470 };
 471
 472 struct intel_uncore_extra_reg {
 473         raw_spinlock_t lock;
 474         u64 config, config1, config2;
 475         atomic_t ref;
 476 };
 477
 478 struct intel_uncore_box {
 479         int phys_id;
 480         int n_active;   /* number of active events */
 481         int n_events;
 482         int cpu;        /* cpu to collect events */
 483         unsigned long flags;
 484         atomic_t refcnt;
 485         struct perf_event *events[UNCORE_PMC_IDX_MAX];
 486         struct perf_event *event_list[UNCORE_PMC_IDX_MAX];
 487         unsigned long active_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
 488         u64 tags[UNCORE_PMC_IDX_MAX];
 489         struct pci_dev *pci_dev;
 490         struct intel_uncore_pmu *pmu;
 491         struct hrtimer hrtimer;
 492         struct list_head list;
 493         struct intel_uncore_extra_reg shared_regs[0];
 494 };
 495
 496 #define UNCORE_BOX_FLAG_INITIATED       0
 497
 498 struct uncore_event_desc {
 499         struct kobj_attribute attr;
 500         const char *config;
 501 };
 502
 503 #define INTEL_UNCORE_EVENT_DESC(_name, _config)                 \
 504 {                                                               \
 505         .attr   = __ATTR(_name, 0444, uncore_event_show, NULL), \
 506         .config = _config,                                      \
 507 }
 508
 509 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format)                 \
 510 static ssize_t __uncore_##_var##_show(struct kobject *kobj,             \
 511                                 struct kobj_attribute *attr,            \
 512                                 char *page)                             \
 513 {                                                                       \
 514         BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);                     \
 515         return sprintf(page, _format "\n");                             \
 516 }                                                                       \
 517 static struct kobj_attribute format_attr_##_var =                       \
 518         __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
 519
 520
 521 static ssize_t uncore_event_show(struct kobject *kobj,
 522                                 struct kobj_attribute *attr, char *buf)
 523 {
 524         struct uncore_event_desc *event =
 525                 container_of(attr, struct uncore_event_desc, attr);
 526         return sprintf(buf, "%s", event->config);
 527 }
 528
 529 static inline unsigned uncore_pci_box_ctl(struct intel_uncore_box *box)
 530 {
 531         return box->pmu->type->box_ctl;
 532 }
 533
 534 static inline unsigned uncore_pci_fixed_ctl(struct intel_uncore_box *box)
 535 {
 536         return box->pmu->type->fixed_ctl;
 537 }
 538
 539 static inline unsigned uncore_pci_fixed_ctr(struct intel_uncore_box *box)
 540 {
 541         return box->pmu->type->fixed_ctr;
 542 }
 543
 544 static inline
 545 unsigned uncore_pci_event_ctl(struct intel_uncore_box *box, int idx)
 546 {
 547         return idx * 4 + box->pmu->type->event_ctl;
 548 }
 549
 550 static inline
 551 unsigned uncore_pci_perf_ctr(struct intel_uncore_box *box, int idx)
 552 {
 553         return idx * 8 + box->pmu->type->perf_ctr;
 554 }
 555
 556 static inline unsigned uncore_msr_box_offset(struct intel_uncore_box *box)
 557 {
 558         struct intel_uncore_pmu *pmu = box->pmu;
 559         return pmu->type->msr_offsets ?
 560                 pmu->type->msr_offsets[pmu->pmu_idx] :
 561                 pmu->type->msr_offset * pmu->pmu_idx;
 562 }
 563
 564 static inline unsigned uncore_msr_box_ctl(struct intel_uncore_box *box)
 565 {
 566         if (!box->pmu->type->box_ctl)
 567                 return 0;
 568         return box->pmu->type->box_ctl + uncore_msr_box_offset(box);
 569 }
 570
 571 static inline unsigned uncore_msr_fixed_ctl(struct intel_uncore_box *box)
 572 {
 573         if (!box->pmu->type->fixed_ctl)
 574                 return 0;
 575         return box->pmu->type->fixed_ctl + uncore_msr_box_offset(box);
 576 }
 577
 578 static inline unsigned uncore_msr_fixed_ctr(struct intel_uncore_box *box)
 579 {
 580         return box->pmu->type->fixed_ctr + uncore_msr_box_offset(box);
 581 }
 582
 583 static inline
 584 unsigned uncore_msr_event_ctl(struct intel_uncore_box *box, int idx)
 585 {
 586         return box->pmu->type->event_ctl +
 587                 (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) +
 588                 uncore_msr_box_offset(box);
 589 }
 590
 591 static inline
 592 unsigned uncore_msr_perf_ctr(struct intel_uncore_box *box, int idx)
 593 {
 594         return box->pmu->type->perf_ctr +
 595                 (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) +
 596                 uncore_msr_box_offset(box);
 597 }
 598
 599 static inline
 600 unsigned uncore_fixed_ctl(struct intel_uncore_box *box)
 601 {
 602         if (box->pci_dev)
 603                 return uncore_pci_fixed_ctl(box);
 604         else
 605                 return uncore_msr_fixed_ctl(box);
 606 }
 607
 608 static inline
 609 unsigned uncore_fixed_ctr(struct intel_uncore_box *box)
 610 {
 611         if (box->pci_dev)
 612                 return uncore_pci_fixed_ctr(box);
 613         else
 614                 return uncore_msr_fixed_ctr(box);
 615 }
 616
 617 static inline
 618 unsigned uncore_event_ctl(struct intel_uncore_box *box, int idx)
 619 {
 620         if (box->pci_dev)
 621                 return uncore_pci_event_ctl(box, idx);
 622         else
 623                 return uncore_msr_event_ctl(box, idx);
 624 }
 625
 626 static inline
 627 unsigned uncore_perf_ctr(struct intel_uncore_box *box, int idx)
 628 {
 629         if (box->pci_dev)
 630                 return uncore_pci_perf_ctr(box, idx);
 631         else
 632                 return uncore_msr_perf_ctr(box, idx);
 633 }
 634
 635 static inline int uncore_perf_ctr_bits(struct intel_uncore_box *box)
 636 {
 637         return box->pmu->type->perf_ctr_bits;
 638 }
 639
 640 static inline int uncore_fixed_ctr_bits(struct intel_uncore_box *box)
 641 {
 642         return box->pmu->type->fixed_ctr_bits;
 643 }
 644
 645 static inline int uncore_num_counters(struct intel_uncore_box *box)
 646 {
 647         return box->pmu->type->num_counters;
 648 }
 649
 650 static inline void uncore_disable_box(struct intel_uncore_box *box)
 651 {
 652         if (box->pmu->type->ops->disable_box)
 653                 box->pmu->type->ops->disable_box(box);
 654 }
 655
 656 static inline void uncore_enable_box(struct intel_uncore_box *box)
 657 {
 658         if (box->pmu->type->ops->enable_box)
 659                 box->pmu->type->ops->enable_box(box);
 660 }
 661
 662 static inline void uncore_disable_event(struct intel_uncore_box *box,
 663                                 struct perf_event *event)
 664 {
 665         box->pmu->type->ops->disable_event(box, event);
 666 }
 667
 668 static inline void uncore_enable_event(struct intel_uncore_box *box,
 669                                 struct perf_event *event)
 670 {
 671         box->pmu->type->ops->enable_event(box, event);
 672 }
 673
 674 static inline u64 uncore_read_counter(struct intel_uncore_box *box,
 675                                 struct perf_event *event)
 676 {
 677         return box->pmu->type->ops->read_counter(box, event);
 678 }
 679
 680 static inline void uncore_box_init(struct intel_uncore_box *box)
 681 {
 682         if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) {
 683                 if (box->pmu->type->ops->init_box)
 684                         box->pmu->type->ops->init_box(box);
 685         }
 686 }
 687
 688 static inline bool uncore_box_is_fake(struct intel_uncore_box *box)
 689 {
 690         return (box->phys_id < 0);
 691 }